最近在用DDPG算法解决路径规划问题，但是不知道为什么actor网络输出的多维动作向量值总是在0附近大概都是1e-3数量级，而且每个step的变化都不大，导致转换后的动作（比如移动方向、速度、发射功率等）都很小
actor网络的loss值打出来是有在变化的，网络参数也是有变化的，神经网络的输入也得提前调到了同一数量级（1-10），BN层和LN层都尝试加过了，真的调试了好久，也还是不知道到底是哪里出了问题，神经网络模型长这样（加了LN层的），希望有小伙伴可以告诉我还有可能是哪里的问题啊
```python
[array([ 0.00363818, 0.0018692 , 0.00566376, -0.00982124, -0.01358668],
dtype=float32), array([ 0.00291705, 0.00226895, 0.00478148, -0.00890055, -0.01326749],
dtype=float32), array([ 0.01321111, -0.00697791, 0.0161714 , -0.01741296, -0.02361217],
dtype=float32)]
[array([ 0.00532129, 0.00018366, 0.00718524, -0.01139973, -0.01506326],
dtype=float32), array([ 0.0043016 , 0.00165614, 0.00569433, -0.0099882 , -0.01458266],
dtype=float32), array([ 0.01287621, -0.00733752, 0.01562952, -0.01715881, -0.02311403],
dtype=float32)]
[array([ 0.00528781, 0.00021858, 0.00714553, -0.01137979, -0.01502699],
dtype=float32), array([ 0.00429485, 0.00165806, 0.00569154, -0.00998067, -0.01457683],
dtype=float32), array([ 0.01285914, -0.00732083, 0.01560804, -0.01715397, -0.02309367],
dtype=float32)]
[array([ 0.00401041, 0.00186404, 0.00624963, -0.00991879, -0.01420762],
dtype=float32), array([ 0.00283949, 0.00227825, 0.00515016, -0.00914116, -0.01310303],
dtype=float32), array([ 0.01349892, -0.0071977 , 0.01686332, -0.01759761, -0.02409484],
dtype=float32)]
[array([ 0.00400529, 0.00187367, 0.00623996, -0.00991626, -0.01420152],
dtype=float32), array([ 0.00282853, 0.0022863 , 0.00514006, -0.00913269, -0.0130916 ],
dtype=float32), array([ 0.01344847, -0.00715144, 0.01680301, -0.01756832, -0.02403875],
dtype=float32)]
```python
class MLPNetwork(nn.Module):
"""
MLP network (can be used as value or policy)
"""
def __init__(self, input_dim, out_dim, hidden_dim1=128, hidden_dim2=64, nonlin=F.relu,
constrain_out=False, norm_in=False, discrete_action=True):
"""
Inputs:
input_dim (int): Number of dimensions in input
out_dim (int): Number of dimensions in output
hidden_dim (int): Number of hidden dimensions
nonlin (PyTorch function): Nonlinearity to apply to hidden layers
"""
super(MLPNetwork, self).__init__()
if norm_in: # normalize inputs
self.in_fn = nn.BatchNorm1d(input_dim) # 批归一化
self.in_fn.weight.data.fill_(1)
self.in_fn.bias.data.fill_(0)
else:
self.in_fn = lambda x: x
# 层归一化
self.fc1 = nn.Linear(input_dim, hidden_dim1)
self.ln1 = nn.LayerNorm(hidden_dim1)
self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
self.ln2 = nn.LayerNorm(hidden_dim2)
self.fc3 = nn.Linear(hidden_dim2, out_dim)
self.nonlin = nonlin
if constrain_out and not discrete_action:
# initialize small to prevent saturation
self.fc3.weight.data.uniform_(-3e-3, 3e-3)
self.fc3.bias.data.uniform_(-3e-3, 3e-3)
self.out_fn = torch.tanh
else: # logits for discrete action (will softmax later)
self.out_fn = lambda x: x
def forward(self, X):
"""
Inputs:
X (PyTorch Matrix): Batch of observations
Outputs:
out (PyTorch Matrix): Output of network (actions, values, etc)
"""
h1 = self.nonlin(self.ln1(self.fc1(self.in_fn(X))))
h2 = self.nonlin(self.ln2(self.fc2(h1)))
out = self.out_fn(self.fc3(h2))
return out
