Parameter initialization

Use numpy initialization

import numpy as np
import torch
from torch import nn

# Define a Sequential Model
net1=nn.Sequential(
    nn.Linear(30,40),
    nn.ReLU(),
    nn.Linear(40,50),
    nn.ReLU(),
    nn.Linear(50,10)
)

w1=net1[0].weight   # The parameters of the first layer
b1=net1[0].bias

print(w1)

net1[0].weight.data=torch.from_numpy(np.random.uniform(3,5,size=(40,30))) # Try to change the parameters of the first layer

print(net1[0].weight)

 

for layer in net1:     # Yes Sequential Parameter initialization
    if isinstance(layer,nn.Linear):
        param_shape=layer.weight.shape
        layer.weight.data = torch.from_numpy(np.random.normal(0, 0.5,
size=param_shape))  

about Moudule Parameter initialization , In fact, it's very simple , If you want to initialize one of these layers , Can be directly like Sequential The same for its Tensor To redefine , The only difference is , If you want to access... In a circular way , Two properties need to be introduced ,children and modules, Here are some examples to illustrate .

class sim_net(nn.Module):
    def __init__(self):
        super(sim_net, self).__init__()
        self.l1 = nn.Sequential(
            nn.Linear(30, 40),
            nn.ReLU()
)
        self.l1[0].weight.data = torch.randn(40, 30) # ?????????
        self.l2 = nn.Sequential(
            nn.Linear(40, 50),
            nn.ReLU()
)
        self.l3 = nn.Sequential(
            nn.Linear(50, 10),
            nn.ReLU()
)
    def forward(self, x):
        x = self.l1(x)
        x = self.l2(x)
        x = self.l3(x)
        return x

net2=sim_net()

for i in net2.children():
    print(i)

for i in net2.modules():
    print(i)

 

Through the example above , We can see that , children Only the first level of model definition will be accessed , Because the above model defines Sequential, So only three... Will be visited Sequential, and Modules Will access the final structure , Take the example above ,moudles Not only visited sequential, Also visited Sequential Inside , This is very convenient for us to initialize

for layer in net2.modules():
    if isinstance(layer, nn.Linear):
        param_shape = layer.weight.shape
        layer.weight.data = torch.from_numpy(np.random.normal(0, 0.5,
size=param_shape))