Original address: Explain ModuleList and Sequential in PyTorch in detail
torch.nn.Module()
import torch.nn as nn import torch.nn.functional as F class Model(nn.Module): def __init__(self): super(Model, self).__init__() self.conv1 = nn.Conv2d(3, 32, 3) self.conv2 = nn.Conv2d(32, 10, 1) def forward(self, x): x = F.relu(self.conv1(x) return F.relu(self.conv2(x))
torch.nn.Sequential(*args)
- nn. The modules in sequential are arranged in order, so the output size of the previous module must be consistent with the input size of the next module.
- Each module adopts the default naming method (by sequence number 0, 1, 2,...). You can also use OrderedDict to specify the name of each module
#Using Sequential
model = nn.Sequential(
nn.Conv2d(3, 32, 3),
nn.ReLU()
nn.Conv2d(32, 10, 1)
nn.ReLu()
)
# Using Sequential and OrderedDict
model = nn.Sequential(OrderedDict([
('conv1', nn.Conv2d(3, 32, 3)),
('ReLu1', nn.ReLu()),
('conv2', nn.Conv2d(32, 10, 1)),
('ReLu2', nn.ReLu())
]))
torch.ModuleList(module=None)
- nn.ModuleList is a container that stores different modules and automatically adds the parameters of each module to the network
- You can put any NN Subclasses of module (such as nn.Conv2d, nn.Linear, etc.) are added to this list. The method is the same as Python's own list, which is nothing more than extend, append and other operations
- But different from the general list, add to NN The module in the modulelist will be automatically registered to the whole network, and the parameters of the module will be automatically added to the whole network. If you use python's list, there will be a problem.
class net_modlist(nn.Module):
def __init__(self):
super(net_modlist, self).__init__()
self.modlist = nn.ModuleList([
nn.Conv2d(1, 20, 5),
nn.ReLU(),
nn.Conv2d(20, 64, 5),
nn.ReLU()
])
def forward(self, x):
for m in self.modlist:
x = m(x)
return x
net_modlist = net_modlist()
print(net_modlist)
#net_modlist(
# (modlist): ModuleList(
# (0): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1))
# (1): ReLU()
# (2): Conv2d(20, 64, kernel_size=(5, 5), stride=(1, 1))
# (3): ReLU()
# )
#)
for param in net_modlist.parameters():
print(type(param.data), param.size())
#<class 'torch.Tensor'> torch.Size([20, 1, 5, 5])
#<class 'torch.Tensor'> torch.Size([20])
#<class 'torch.Tensor'> torch.Size([64, 20, 5, 5])
#<class 'torch.Tensor'> torch.Size([64])
Use Python's own list
class net_modlist(nn.Module):
def __init__(self):
super(net_modlist, self).__init__()
self.modlist = [
nn.Conv2d(1, 20, 5),
nn.ReLU(),
nn.Conv2d(20, 64, 5),
nn.ReLU()
]
def forward(self, x):
for m in self.modlist:
x = m(x)
return x
net_modlist = net_modlist()
print(net_modlist)
#net_modlist()
for param in net_modlist.parameters():
print(type(param.data), param.size())
#None
The convolution layers added using Python's list and their parameters are not automatically registered in our network. We can use forward to calculate the output. However, when training with its instantiated network, because the parameters of these layers are not in the whole network, the network parameters will not be updated, that is, they cannot be trained.
nn.Sequential and NN Differences between modulelist
Differences 1
nn.Sequential internally implements the forward function, so you don't need to write the forward function. And NN Modulelist does not implement the internal forward function.
#Example 1: This is an example from official documents
seq = nn.Sequential(
nn.Conv2d(1,20,5),
nn.ReLU(),
nn.Conv2d(20,64,5),
nn.ReLU()
)
print(seq)
# Sequential(
# (0): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1))
# (1): ReLU()
# (2): Conv2d(20, 64, kernel_size=(5, 5), stride=(1, 1))
# (3): ReLU()
# )
#Input the above seq
input = torch.randn(16, 1, 20, 20)
print(seq(input))
#torch.Size([16, 64, 12, 12])
#Or: NN 2 Module class, you need to write the forward function
class net1(nn.Module):
def __init__(self):
super(net1, self).__init__()
self.seq = nn.Sequential(
nn.Conv2d(1,20,5),
nn.ReLU(),
nn.Conv2d(20,64,5),
nn.ReLU()
)
def forward(self, x):
return self.seq(x)
#Note: the same result can be obtained by using the for loop as follows
#def forward(self, x):
# for s in self.seq:
# x = s(x)
# return x
#Input net1
input = torch.randn(16, 1, 20, 20)
net1 = net1()
print(net1(input).shape)
#torch.Size([16, 64, 12, 12])
For ModuleList
#Example 1: if it is written as follows, an error will be generated
modlist = nn.ModuleList([
nn.Conv2d(1, 20, 5),
nn.ReLU(),
nn.Conv2d(20, 64, 5),
nn.ReLU()
])
print(modlist)
#ModuleList(
# (0): Conv2d(1, 20, kernel_size=(5, 5), stride=(1, 1))
# (1): ReLU()
# (2): Conv2d(20, 64, kernel_size=(5, 5), stride=(1, 1))
# (3): ReLU()
#)
input = torch.randn(16, 1, 20, 20)
print(modlist(input))
#Generate NotImplementedError
#Example 2: write forward function
class net2(nn.Module):
def __init__(self):
super(net2, self).__init__()
self.modlist = nn.ModuleList([
nn.Conv2d(1, 20, 5),
nn.ReLU(),
nn.Conv2d(20, 64, 5),
nn.ReLU()
])
#If this rule is followed here, a notimplemented error will be reported
#def forward(self, x):
# return self.modlist(x)
#Note: you can only use the for loop as follows
def forward(self, x):
for m in self.modlist:
x = m(x)
return x
input = torch.randn(16, 1, 20, 20)
net2 = net2()
print(net2(input).shape)
#torch.Size([16, 64, 12, 12])
Differences 2
nn.Sequential can name each layer using OrderedDict
Differences 3
- nn. The modules in sequential are arranged in order, so it must be ensured that the output size of the previous module is consistent with the input size of the next module.
- nn.ModuleList does not define a network. It just stores different modules together. There is no order between these modules. The network execution order is determined by the forward function.
summary
- Sequential internal module s are ordered, and the forward function has been implemented internally
- The internal modules of ModuleList are out of order. You need to write the forward function yourself. The order of module execution is determined by the forward function
- Code tip: if there are many similar or repeated layers in the network, we will generally consider creating them with a for loop instead of writing them line by line. At this time, we can use ModuleList.
class net4(nn.Module):
def __init__(self):
super(net4, self).__init__()
layers = [nn.Linear(10, 10) for i in range(5)]
self.linears = nn.ModuleList(layers)
def forward(self, x):
for layer in self.linears:
x = layer(x)
return x
net = net4()
print(net)
# net4(
# (linears): ModuleList(
# (0): Linear(in_features=10, out_features=10, bias=True)
# (1): Linear(in_features=10, out_features=10, bias=True)
# (2): Linear(in_features=10, out_features=10, bias=True)
# )
# )