LSTM训练时使用训练集进行测试准确率异常

Question

为什么我训练过程在训练一个LSTM神经网络的时候，train_accuracy和val_accuracy已经达到了0.95，训练完成以后用得到的东西检测我训练用的训练集的时侯，发现准确率反而下降至0.55.按道理如果是过拟合的话不应该很高吗?

这个是训练用的文件

import sys
sys.path.append('../input')

import torch
import torch.optim as optim
import torch.nn as nn
from bighomework.bighomework.data_utils import load_data, TextDataset, collate_fn
from bighomework.bighomework.model import LSTMClassifier
from torch.utils.data import DataLoader
from torchtext.vocab import build_vocab_from_iterator
from bighomework.bighomework.draw_picture import draw_val_accuracies,draw_train_accuracies,draw_train_losses,draw_val_losses
from sklearn.model_selection import train_test_split
import pickle

# 定义超参数
embedding_size = 100
hidden_size = 256
output_size = 2
num_layers = 2
dropout = 0.5
batch_size = 128
epochs = 3
learning_rate = 0.001

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f'Using device: {device}')

# 导入数据并进行分词
train_texts, train_labels = load_data('/kaggle/input/datasets4/datasets4/full_ai.txt','/kaggle/input/datasets4/datasets4/full_human.txt')

# 使用简单的空格分词
def tokenizer(text):
    return text.split()

# 构建词汇表
def yield_tokens(data_iter, tokenizer):
    for text in data_iter:
        yield tokenizer(text)

vocab = build_vocab_from_iterator(yield_tokens(full_texts, tokenizer), specials=["<unk>"])
vocab.set_default_index(vocab["<unk>"])
vocab_size = len(vocab)

with open('/kaggle/working/vocab.pkl', 'wb') as f:
    pickle.dump(vocab, f)

# 划分训练集和验证集
train_texts, val_texts, train_labels, val_labels = train_test_split(train_texts, train_labels, test_size=0.2, random_state=42)

# 创建训练集和验证集的数据集和数据加载器
train_dataset = TextDataset(train_texts, train_labels, vocab, tokenizer)
train_loader = DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn)

val_dataset = TextDataset(val_texts, val_labels, vocab, tokenizer)
val_loader = DataLoader(val_dataset, batch_size=batch_size, collate_fn=collate_fn)
# 导入模型
model = LSTMClassifier(vocab_size, embedding_size, hidden_size, output_size, num_layers, dropout).to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

# 训练和验证
train_losses = []
train_accuracies = []
val_losses = []
val_accuracies = []

print(f'Number of batches per epoch: {len(train_loader)}')
for epoch in range(epochs):
    # 训练阶段
    model.train()
    epoch_train_loss = 0
    correct_train_predictions = 0
    total_train_samples = 0
    i=0
    for texts, labels, lengths in train_loader:
 #       print("train:", i+1, "of", len(train_loader), "batches")
        texts, labels, lengths = texts.to(device), labels.to(device), lengths.to(device)
        optimizer.zero_grad() # 清除之前计算的梯度。
        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # 对模型的所有参数进行梯度裁剪，防止梯度爆炸。
        predictions = model(texts, lengths).squeeze(1)   # 使用模型对输入的文本进行预测    
        loss = criterion(predictions, labels) # 计算预测结果与实际标签之间的损失。
        loss.backward()     # 反向传播，计算梯度
        optimizer.step()     # 更新模型参数。  
        epoch_train_loss += loss.item()       
        _, predicted_labels = torch.max(predictions, 1)
        correct_train_predictions += (predicted_labels == labels).sum().item()
        total_train_samples += labels.size(0)
        i+=1
    avg_train_loss = epoch_train_loss / len(train_loader)
    train_accuracy = correct_train_predictions / total_train_samples
    train_losses.append(avg_train_loss)
    train_accuracies.append(train_accuracy)
    
    print(f'Epoch {epoch + 1}/{epochs}, Train Loss: {avg_train_loss:.4f}, Train Accuracy: {train_accuracy:.4f}')
    
    # 验证阶段
    model.eval()
    epoch_val_loss = 0
    correct_val_predictions = 0
    total_val_samples = 0

    with torch.no_grad():
        for texts, labels, lengths in val_loader:
            texts, labels, lengths = texts.to(device), labels.to(device), lengths.to(device)
            predictions = model(texts, lengths).squeeze(1)          
            loss = criterion(predictions, labels)         
            epoch_val_loss += loss.item()          
            _, predicted_labels = torch.max(predictions, 1)
            correct_val_predictions += (predicted_labels == labels).sum().item()
            total_val_samples += labels.size(0)   
    avg_val_loss = epoch_val_loss / len(val_loader)
    val_accuracy = correct_val_predictions / total_val_samples
    val_losses.append(avg_val_loss)
    val_accuracies.append(val_accuracy)
    
    print(f'Epoch {epoch + 1}/{epochs}, Val Loss: {avg_val_loss:.4f}, Val Accuracy: {val_accuracy:.4f}')

# 保存模型
torch.save(model.state_dict(), '/kaggle/working/lstm_classifier6.pth')

# 绘制损失和准确率曲线
draw_train_losses(epochs, train_losses)
draw_val_losses(epochs,val_losses)
draw_train_accuracies(epochs, train_accuracies)
draw_val_accuracies(epochs,val_accuracies)

这个是检测用的文件

import sys
sys.path.append('../input')

import torch
from bighomework.bighomework.model import LSTMClassifier
from bighomework.bighomework.data_utils import TextDataset, collate_fn, load_data
from torch.utils.data import DataLoader
from torchtext.vocab import build_vocab_from_iterator
import pandas as pd
import numpy as np
import pickle

# 定义超参数
embedding_size = 100
hidden_size = 256
output_size = 2
num_layers = 2
dropout = 0.5
batch_size = 128
epochs = 50
learning_rate = 0.001

# 检查是否有可用的GPU设备
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f'Using device: {device}')

# 使用简单的空格分词
def tokenizer(text):
    return text.split()

# 直接生成分词后的迭代器
def yield_tokens(data_iter, tokenizer):
    for text in data_iter:
        yield tokenizer(text)

def load_vocab(vocab_path):
    with open(vocab_path, 'rb') as f:
        vocab = pickle.load(f)
    return vocab

# 加载模型
def load_model(vocab_size, embedding_size, hidden_size, output_size, num_layers, dropout, model_path):
    model = LSTMClassifier(vocab_size, embedding_size, hidden_size, output_size, num_layers, dropout)
    model.load_state_dict(torch.load(model_path))
    model.to(device)  # 将模型移动到GPU上
    return model

def detect_AI(text, model, vocab, tokenizer):
    # 对文本进行分词和编码
    tokens = tokenizer(text)
    indexed = [vocab[token] for token in tokens]
    tensor = torch.LongTensor(indexed).to(device)  # 将数据移动到GPU上
    tensor = tensor.unsqueeze(0)  # 添加 batch 维度

    # 使用模型进行预测
    model.eval()
    with torch.no_grad():
        output = model(tensor, torch.tensor([len(tensor)], device=device))  # 确保长度tensor也在同样设备上
    
    # 获取预测结果概率
    probabilities = torch.nn.functional.softmax(output, dim=1)[0]
    
    return probabilities[0]

if __name__ == "__main__":
    vocab_path = "/kaggle/working/vocab2.pkl"  # 替换为实际的路径
    vocab = load_vocab(vocab_path)
    vocab_size = len(vocab)
    # 加载模型
    model_path = "/kaggle/working/lstm_classifier8.pth"
    model = load_model(vocab_size, embedding_size, hidden_size, output_size, num_layers, dropout, model_path)

    with open('/kaggle/input/datasets4/datasets4/test_ai.txt','r',encoding='utf-8') as file1,open('/kaggle/input/datasets4/datasets4/test_human.txt','r',encoding='utf-8') as file2:
        ai_lines=file1.readlines()
        human_lines=file2.readlines()
        i,score=0,0
        for line in ai_lines:
            res=detect_AI(line,model,vocab,tokenizer)
            if res>0.5:
                score+=1
            i+=1
        print(i,score)
        j,right=0,0
        for line in human_lines:
            res=detect_AI(line,model,vocab,tokenizer)
            if res<0.5:
                right+=1
            j+=1
        print(j,right)
        print("综合准确率：",(right+score)/(i+j))
    
    
    with open('/kaggle/input/datasets4/datasets4/train_ai.txt','r',encoding='utf-8') as file1,open('/kaggle/input/datasets4/datasets4/train_human.txt','r',encoding='utf-8') as file2:
        ai_lines=file1.readlines()
        human_lines=file2.readlines()
        i,score=0,0
        for line in ai_lines:
            res=detect_AI(line,model,vocab,tokenizer)
            if res>0.5:
                score+=1
            i+=1
        print(i,score)
        j,right=0,0
        for line in human_lines:
            res=detect_AI(line,model,vocab,tokenizer)
            if res<0.5:
                right+=1
            j+=1
        print(j,right)
        print("综合准确率：",(right+score)/(i+j))
    
    # 读取 test.csv 文件
    test_df = pd.read_csv("/kaggle/input/llm-detect-ai-generated-text/test_essays.csv")
    
    results = []
    for index, row in test_df.iterrows():
        text = row['text']
        result = detect_AI(text, model, vocab, tokenizer)
        results.append(result)
    
    # 将张量移动到 CPU 并转换为 NumPy 数组
    results_np = [tensor.cpu().numpy() for tensor in results]

    # 假设 'test_df' 是包含 'id' 列的 DataFrame
    submission_df = pd.DataFrame({'id': test_df['id'], 'label': results_np})

    # 写入 CSV 文件
    submission_df.to_csv("/kaggle/working/submission.csv", index=False)

    print("结果已保存到 submission.csv")

Answer 1

以下内容由CHATGPT及阿里嘎多学长共同生成、有用望采纳：

---

从你提供的信息来看，你的LSTM模型在训练集和验证集上都达到了0.95的准确率，但在测试训练集时准确率下降到了0.55，这确实是一个非常不寻常的现象。通常，过拟合会导致模型在训练集上表现非常好，但在新数据上表现不佳。然而，你的情况似乎是模型在训练集上的表现也不佳。以下是一些可能的原因和建议：

1. 数据预处理不一致：确保训练、验证和测试使用的数据经过了相同的预处理步骤。如果数据在不同阶段的处理方式不同，可能会导致模型表现不佳。

2. 模型评估方式：检查你的模型评估代码，确保在评估时使用了正确的方法。例如，确保在评估时没有使用model.train()而是使用了model.eval()，并且在计算准确率时没有使用梯度下降。

3. 模型保存和加载：确保模型在训练完成后正确保存，并且在测试时正确加载了模型的权重。检查torch.save和torch.load的使用是否正确。

4. 批次大小和数据分批：检查你的数据加载器是否正确地将数据分批，特别是在测试时。如果批次大小或数据分批的方式与训练时不同，可能会影响模型的表现。

5. 评估指标：确保你使用的评估指标是正确的。在LSTM模型中，通常使用分类准确率作为评估指标，但如果你的任务是回归或其他类型的预测，可能需要使用不同的指标。

6. 模型复杂性：如果模型过于复杂，可能会导致在训练集上的过拟合，但在测试集上表现不佳。尝试减少模型的复杂性，比如减少层数或神经元数量。

7. 数据集的分布：检查训练集和测试集的数据分布是否一致。如果分布差异很大，模型可能无法很好地泛化到测试集。

8. 随机性因素：神经网络训练过程中存在一定的随机性，比如权重初始化和数据洗牌。尝试重新训练模型几次，看看问题是否仍然存在。

9. 代码错误：仔细检查你的代码，特别是数据处理和模型预测的部分，以确保没有逻辑错误或实现错误。

10. 调试和日志记录：增加更多的日志记录和调试语句，以便更好地理解模型在训练和测试过程中的行为。

如果上述建议都不能解决问题，你可能需要更深入地分析模型的行为，比如通过可视化训练过程中的损失和准确率曲线，或者使用模型解释性工具来理解模型的预测。

最后，由于你没有提供具体的参考链接，我无法提供具体的参考链接。但是，上述建议是基于一般的神经网络训练和调试实践。希望这些建议能帮助你解决问题。如果需要进一步的帮助，请随时提问。