pytorch训练过程中卷积层和loss输出为nan

Question

问题遇到的现象和发生背景

pytorch卷积神经网络训练一段时间后，卷积层输出nan，loss，预测值和标签都变为nan

问题相关代码，请勿粘贴截图

Y-P=tensor([[0.5276, 0.4724],
        [0.5447, 0.4553],
        [0.5436, 0.4564],
        [0.5342, 0.4658],
        [0.5341, 0.4659],
        [0.5402, 0.4598],
        [0.5654, 0.4346],
        [0.5144, 0.4856],
        [0.5525, 0.4475],
        [0.5330, 0.4670],
        [0.5642, 0.4358],
        [0.5294, 0.4706],
        [0.4924, 0.5076],
        [0.5127, 0.4873],
        [0.5457, 0.4543],
        [0.5342, 0.4658]], grad_fn=<SoftmaxBackward0>),Tar=tensor([0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1])
LOSS: 0.6749879717826843
Y-P=tensor([[0.5395, 0.4605],
        [0.5549, 0.4451],
        [0.5141, 0.4859],
        [0.5791, 0.4209],
        [0.5541, 0.4459],
        [0.5286, 0.4714],
        [0.5368, 0.4632],
        [0.5153, 0.4847],
        [0.5218, 0.4782],
        [0.5556, 0.4444],
        [0.5422, 0.4578],
        [0.5571, 0.4429],
        [0.5040, 0.4960],
        [0.5531, 0.4469],
        [0.5329, 0.4671],
        [0.5285, 0.4715]], grad_fn=<SoftmaxBackward0>),Tar=tensor([1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1])
LOSS: 0.6953127384185791
Y-P=tensor([[nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan]], grad_fn=<SoftmaxBackward0>),Tar=tensor([0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0])
LOSS: nan
Y-P=tensor([[nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan],
        [nan, nan]], grad_fn=<SoftmaxBackward0>),Tar=tensor([0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1])
LOSS: nan

运行结果及报错内容

我的解答思路和尝试过的方法

改变学习率后没有效果

我想要达到的结果

程序可以正常运行，不在输出nan

Answer 1

你的训练数据集有问题吗?检查了吗?比如数据集标注的格式啥的