lm-evaluation-harness框架该怎么使用

Question

问下如何使用 https://github.com/EleutherAI/lm-evaluation-harness 的框架对自己实现的模型进行评估。比如说自己现在就实现了一个LSTM算法。代码如下，那么如何能调用这个框架对这个模型进行评估。始终没有查到相关的数据集和API接口该怎么调用

class SimpleLstmNetWork(nn.Module):
    def __init__(self, taskName, isBatchFirst, trainDataDim, labelDataDim, hiddenDim, layerNum, batchSize):
        # 初始化父类
        super(SimpleLstmNetWork, self).__init__()
        self.taskName = taskName
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        # 定义 LSTM 模型的参数
        self.trainDataDim = trainDataDim
        self.hiddenDim = hiddenDim
        self.layerNum = layerNum
        self.batchSize = batchSize
        # 定义 LSTM 层
        self.trainModule = nn.LSTM(trainDataDim, hiddenDim, layerNum, batch_first = isBatchFirst).to(self.device)
        # 定义输出层
        self.fullConnectLayer = nn.Linear(hiddenDim, labelDataDim).to(self.device)

    def forward(self, inputData):
        # LSTM 前向传播
        out, _ = self.trainModule(inputData)
        # 模型输出
        out = self.fullConnectLayer(out[:, -1, :])
        return out
        
    def SetCriterion(self, func):
        self.criterion = func
        
    def SetOptimizer(self, func):
        self.optimizer = func

    def SetLstmTrainData(self, inputData, labelData):
        # 将数据转换为 PyTorch Dataset 对象并放到 GPU 上
        data = TensorDataset(inputData.to(self.device), labelData.to(self.device))
        # 加载数据并自动进行批处理
        self.dataloader = DataLoader(data, batch_size = self.batchSize, shuffle = False)

    def TrainLstmModule(self, epochNum, statPeriod):
        # 模型训练
        for epoch in range(epochNum):
            for trainData, labelData in self.dataloader:
                self.optimizer.zero_grad()
                output = self.forward(trainData)
                loss = self.criterion(output, labelData)
                #torch.autograd.set_detect_anomaly(True)
                loss.backward()
                #torch.autograd.set_detect_anomaly(True)
                self.optimizer.step()
            if (epoch + 1) % statPeriod == 0:
                print(f"taskName = {self.taskName}, Epoch[{epoch + 1}/{epochNum}], loss:{loss.item()}")

    def GetLstmModuleTrainRst(self, verifyData):
        #模型预测
        with torch.no_grad():
            verifyData = verifyData.to(self.device)
            output = self.forward(verifyData)
            return output.cpu()
            
def HandleSimpleLstmNetWorkProcess(taskName, isBatchFirst, trainData, labelData, trainDataDim, labelDataDim, hiddenDim, layerNum, batchSize, epochNum, learnRate, weightDecay, statPeriod, modulePath):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    #print(f"trainData = {trainData.shape}, labelData = {labelData.shape}, trainDataDim = {trainDataDim}, labelDataDim = {labelDataDim}, hiddenDim = {hiddenDim}, batchSize = {batchSize}")
    model = SimpleLstmNetWork(taskName, isBatchFirst, trainDataDim, labelDataDim, hiddenDim, layerNum, batchSize).to(device)
    if os.path.exists(modulePath):
        checkpoint = torch.load(modulePath)
        model.load_state_dict(checkpoint['model_state_dict'])
        return model
    else:
        print(f"HandleSimpleLstmNetWorkProcess path not exist, path = {modulePath}")
        model.SetCriterion(nn.MSELoss())
        #model.SetCriterion(nn.CrossEntropyLoss())
        model.SetOptimizer(torch.optim.Adam(model.parameters(), lr = learnRate, weight_decay = weightDecay))
        model.to(device)
        model.SetLstmTrainData(trainData, labelData)
        model.TrainLstmModule(epochNum, statPeriod)
        torch.save({'model_state_dict': model.state_dict()}, modulePath)
        return model

Answer 1

引自免费微信小程序：皆我百晓生

要使用lm-evaluation-harness框架对自己的模型进行评估，你需要按照以下步骤操作：

准备工作

首先，确保已经安装了lm-evaluation-harness库，并且已将所需的依赖项添加到了你的项目中。

pip install lm-evaluation-harness

实现模型

创建一个SimpleLstmNetWork类，继承自nn.Module（神经网络模块）。在构造函数中初始化模型的各种组件。

import torch.nn as nn
from torch import optim

# 定义模型类
class SimpleLstmNetWork(nn.Module):
    def __init__(self, task_name, is_batch_first, train_data_dim, label_data_dim, hidden_dim, layer_num, batch_size):
        super().__init__()

        self.task_name = task_name
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        # LSTM 参数
        self.train_data_dim = train_data_dim
        self.hidden_dim = hidden_dim
        self.layer_num = layer_num
        self.batch_size = batch_size

        # LSTM
        self.train_module = nn.LSTM(train_data_dim, hidden_dim, layer_num, batch_first=is_batch_first).to(self.device)

        # 输出层
        self.full_connect_layer = nn.Linear(hidden_dim, label_data_dim).to(self.device)

    def forward(self, input_data):
        # LSTM 前向传播
        out, _ = self.train_module(input_data)
        # 模型输出
        out = self.full_connect_layer(out[:, -1, :])
        return out
    
    def set_criterion(self, criterion):
        self.criterion = criterion

    def set_optimizer(self, optimizer):
        self.optimizer = optimizer

    def set_lstm_train_data(self, input_data, label_data):
        # 将数据转换为 PyTorch Dataset 对象并放到 GPU 上
        data = TensorDataset(input_data.to(self.device), label_data.to(self.device))
        # 加载数据并自动进行批处理
        self.dataloader = DataLoader(data, batch_size=self.batch_size, shuffle=False)

    def train_lstm_module(self, epoch_num, stat_period):
        # 模型训练
        for epoch in range(epoch_num):
            for input_data, label_data in self.dataloader:
                self.optimizer.zero_grad()
                output = self.forward(input_data)
                loss = self.criterion(output, label_data)
                #torch.autograd.set_detect_anomaly(True)
                loss.backward()
                #torch.autograd.set_detect_anomaly(True)
                self.optimizer.step()
            if (epoch + 1) % stat_period == 0:
                print(f"Task Name: {self.task_name}, Epoch [{epoch + 1}/{epoch_num}], Loss: {loss.item()}")

    def get_lstm_module_train_rst(self, verify_data):
        # 模型预测
        with torch.no_grad():
            verify_data = verify_data.to(self.device)
            output = self.forward(verify_data)
            return output.cpu()

    def handle_simple_lstm_net_work_process(self, task_name, is_batch_first, train_data, label_data, train_data_dim, label_data_dim, hidden_dim, layer_num, batch_size, epoch_num, learning_rate, weight_decay, stat_period, module_path):
        # 设置设备
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        # 创建模型实例
        model = SimpleLstmNetWork(task_name, is_batch_first, train_data_dim, label_data_dim, hidden_dim, layer_num, batch_size).to(device)

        # 加载预训练模型权重
        if os.path.exists(module_path):
            checkpoint = torch.load(module_path)
            model.load_state_dict(checkpoint['model_state_dict'])
            return model
        else:
            print(f"HANDLE_SIMPLE_LSTM_NET_WORK_PROCESS path does not exist, path = {module_path}")
            model.set_criterion(nn.MSELoss())
            model.set_optimizer(torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay))
            model.to(device)
            model.set_lstm_train_data(train_data, label_data)
            model.train_lstm_module(epoch_num, stat_period)
            torch.save({'model_state_dict': model.state_dict()}, module_path)
            return model

调用框架

当你的模型完成训练后，可以调用handle_simple_lstm_net_work_process方法来开始评估过程。

# 示例：评估任务名称为 "my_task" 的模型
model = handle_simple_lstm_net_work_process(
    "my_task",
    True,
    torch.randn((32, 8, 64)),  # 输入数据
    torch.randint(low=0, high=10, size=(32,)).long().unsqueeze(-1),  # 标签数据
    64,  # 输入特征维度
    10,  # 标签类别数
    32,  # 隐藏单元数量
    4,  # LSTM层数
    32,  # 批量大小
    5,  # 学习率
    0.0001,  # 权重衰减系数
    10,  # 统计周期
    "/path/to/model.pth"
)

# 现在你可以使用 `model` 进行其他任务了。

以上就是使用lm-evaluation-harness框架对模型进行评估的基本步骤。根据你的需求调整输入数据、标签数据等参数。