记录训练神经网络过程中常用的权值共享和不同层赋予不同学习率等方法

权值共享

import torch
import torch.nn as nnclass model(nn.Module):def __init__(self):super(model,self).__init__()self.lstm &＃61; nn.LSTM(input_size &＃61; 10,hidden_size &＃61; 5)self.linear &＃61; nn.Linear(input_features &＃61; 5,out_features &＃61; 2)def forward(self,inputdata1,inputdata2):lstm_result1 &＃61; self.lstm (input_data1)lstm_result2 &＃61; self.lstm(inputdata2)output &＃61; self.linear(lstm_result1&＃43;lstm_result2)return output


注释&＃xff1a;在神经网络的训练过程中经常用到两层网络共享权值&＃xff0c;在上述代码片中&＃xff0c;定义神经网络时定义一个lstm模型和一个全连接层&＃xff0c;在前向计算中多次调用lstm层进行计算&＃xff0c;相当于神经网络模型中有两个lstm层&＃xff0c;即计算inputdata1和inputdata2的两个lstm共享权值。
参考&＃xff1a;https://www.cnblogs.com/sdu20112013/p/12132786.html

某些层参数不更新

在查询此类资料时&＃xff0c;在博客中看到模型层中添加了requires_grad &＃61; False后参数仍会训练的问题&＃xff0c;博主并给出了相关解决方法&＃xff0c;这里记录两个感觉使用方便的方法。
更多内容参考&＃xff1a;https://blog.csdn.net/guotong1988/article/details/79739775

import torch
import torch.nn as nnclass model(nn.Module):def __init__(self):super(model,self).__init__()self.lstm &＃61; nn.LSTM(input_size &＃61; 10,hidden_size &＃61; 5)for p in self.parameters():p.requires_grad &＃61; Falseself.linear &＃61; nn.Linear(input_features &＃61; 5,out_features &＃61; 2)def forward(self,inputdata1,inputdata2):lstm_result1 &＃61; self.lstm (input_data1)lstm_result2 &＃61; self.lstm(inputdata2)output &＃61; self.linear(lstm_result1&＃43;lstm_result2)return output


注释&＃xff1a;在不需要参数更新的层后边添加如下代码行&＃xff1a;

for p in self.parameters():p.requires_grad &＃61; False


但是上述方法适用于模型中最初几层都不需训练&＃xff0c;顶层需要训练的情况&＃xff0c;如果出现需要训练和不需要训练的模型层交替出现的时候&＃xff0c;上述方法就无法使用。博主给出了使用范围更广的方法&＃xff1a;

import torch
import torch.nn as nn
from torch.nn import CrossEntropyLossclass model(nn.Module):def __init__(self):super(model,self).__init__()self.lstm1 &＃61; nn.LSTM(input_size &＃61; 10,hidden_size &＃61; 10,requires_grad &＃61; True)self.lstm2 &＃61; nn.LSTM(input_size &＃61; 10,hidden_size &＃61; 5,requires_grad &＃61; False)self.linear &＃61; nn.Linear(input_features &＃61; 5,out_features &＃61; 2,requires_grad &＃61; True)def forward(self,inputdata):lstm_result1 &＃61; self.lstm1(input_data)lstm_result2 &＃61; self.lstm(lstm_result1)output &＃61; self.linear(lstm_result1&＃43;lstm_result2)return output
model &＃61; model()
#人为构造输入和真实标签
input_data &＃61; torch.randn([1,10])#[1,10]代表输入一个样本&＃xff0c;该样本的向量是10维&＃xff0c;此处必须是二位数据
target &＃61; torch.tensor([1],dtype &＃61; torch.long)#输入一个样本时真实标签只有一个&＃xff0c;如果输入是[5,10]&＃xff0c;则真实标签就应该为5个,例如,torch.tensor([0,1,1,1,0])#模型计算&＃xff0c;反向传播
result &＃61; model(input_data)
loss_fc &＃61; CrossEntropyLoss()
loss &＃61; loss_fc(input_data,target)
loss.backward()#优化函数优化
torch.optimizer.SGD(filter(lambda p:p.requires_grad &＃61; True,model.parameters(),lr &＃61; 0.01))


注释&＃xff1a;上述代码片在优化函数部分对参数进行过滤&＃xff0c;只选取requires_grad &＃61; True的参数进行优化更新。

为不同的层赋予不同的学习率

import torch
import torch.nn as nn
from torch.nn import CrossEntropyLossclass model(nn.Module):def __init__(self):super(model,self).__init__()self.lstm &＃61; nn.LSTM(input_size &＃61; 10,hidden_size &＃61; 10,requires_grad &＃61; True)self.linear &＃61; nn.Linear(input_features &＃61; 5,out_features &＃61; 2,requires_grad &＃61; True)def forward(self,inputdata):lstm_result &＃61; self.lstm(input_data)output &＃61; self.linear(lstm_result)return outputmodel &＃61; model()#人为构造输入和真实标签
input_data &＃61; torch.randn([1,10])#[1,10]代表输入一个样本&＃xff0c;该样本的向量是10维&＃xff0c;此处必须是二位数据
target &＃61; torch.tensor([1],dtype &＃61; torch.long)#输入一个样本时真实标签只有一个&＃xff0c;如果输入是[5,10]&＃xff0c;则真实标签就应该为5个,例如,torch.tensor([0,1,1,1,0])#模型计算&＃xff0c;反向传播
result &＃61; model(input_data)
loss_fc &＃61; CrossEntropyLoss()
loss &＃61; loss_fc(input_data,target)
loss.backward()#使用优化函数优化过程中&＃xff0c;为不同的层赋予不同的学习率&＃xff0c;
param_lstm &＃61; [p for p in model.lstm.parameters()]
param_linear &＃61; [p for p in model.linear.parameters()]
params &＃61; [{&＃39;params&＃39;:param_lstm,&＃39;lr&＃39;:0.1},{&＃39;params&＃39;:param_linear,&＃39;lr&＃39;:0.01}]
torch.optimizer.SGD(params)


将两个模型参数的平均值赋予第三个模型

import torch
import torch.nn as nn
from collections import OrderedDict
#创建两个模型
model1 &＃61; nn.Linear(10,10)
model2 &＃61; nn.Linear(10,10)#获取两个模型的平均值
param_dict &＃61; {}
for key in model1.state_dict.keys():#model1.state_dict()输出值为OrderedDict类型param_key &＃61; (model1.state_dict[key] &＃43; model2.state_dict[key]) / 2param_dict[key] &＃61; param_key#将两个模型的平均值转换成OrderedDict类型&＃xff0c;并赋予第三个模型
param_dict &＃61; OrderedDict(param_dict)
model3 &＃61; nn.Linear(10,10)#三个模型的构造必须一致
model3.load_state_dict(param_dict)


输出模型中每个层的梯度

import torch
import torch.nn as nn
from torch.nn import CrossEntropyLossclass model(nn.Module):def __init__(self):super(model,self).__init__()self.lstm &＃61; nn.LSTM(input_size &＃61; 10,hidden_size &＃61; 10,requires_grad &＃61; True)self.linear &＃61; nn.Linear(input_features &＃61; 5,out_features &＃61; 2,requires_grad &＃61; True)def forward(self,inputdata):lstm_result &＃61; self.lstm(input_data)output &＃61; self.linear(lstm_result)return outputmodel &＃61; model()#人为构造输入和真实标签
input_data &＃61; torch.randn([1,10])#[1,10]代表输入一个样本&＃xff0c;该样本的向量是10维&＃xff0c;此处必须是二位数据
target &＃61; torch.tensor([1],dtype &＃61; torch.long)#输入一个样本时真实标签只有一个&＃xff0c;如果输入是[5,10]&＃xff0c;则真实标签就应该为5个,例如,torch.tensor([0,1,1,1,0])#模型计算&＃xff0c;反向传播
result &＃61; model(input_data)
loss_fc &＃61; CrossEntropyLoss()
loss &＃61; loss_fc(input_data,target)
loss.backward()#输出不同层的梯度
print(model.lstm.grad)
print(model.linear.grad)#细分输出不同层权值和偏置的梯度
print(model.lstm.weight.grad)
print(model.lstm.bias.grad)
print(model.linear.weight.grad)
print(model.linear.bias.grad)


查看模型梯度参考&＃xff1a;https://zhuanlan.zhihu.com/p/36121066
后续还需了解如何直接为某层赋予一定的梯度。