使用Hugging Face Lighteval自定义跨语言同义词识别任务报错排查

一、解决TypeError参数错误 #

直接删除run_evaluation.py中PipelineParameters初始化里的custom_task_directory='evaluation'行——该参数不属于PipelineParameters类的定义，是错误配置。

二、调整自定义任务与数据集配置 #

1. 修改evaluation/custom_csi_task.py #

添加任务配置，绑定数据集路径与元信息：

import numpy as np
from lighteval.tasks.lighteval_task import LightevalTask
from lighteval.tasks.requests import Doc
from lighteval.tasks.task_configs import TaskConfig

class CustomCSI(LightevalTask):
    def doc_to_text(self, doc: Doc) -> str:
        return doc["question"]

    def doc_to_target(self, doc: Doc) -> int:
        return doc["choices"].index(doc["answer"])

    def construct_requests(self, doc: Doc, ctx: str) -> list:
        from lighteval.tasks.requests import Request
        return [Request(request_type="loglikelihood", args=(ctx, " " + choice)) for choice in doc["choices"]]

    def process_results(self, doc: Doc, results: list) -> dict:
        prediction_index = np.argmax(results)
        ground_truth_index = self.doc_to_target(doc)
        return {"acc": 1 if prediction_index == ground_truth_index else 0}

# 绑定数据集路径与任务元信息
CSI_TASK_CONFIG = TaskConfig(
    name="custom_csi",
    path="../csi_benchmark_advanced.jsonl",  # 相对evaluation目录的数据集路径，也可写绝对路径
    task_type="multiple_choice",
    metrics=["acc"],
    stop_sequence=["\n"],
)

# 注册任务构造函数
def custom_csi():
    return CustomCSI(config=CSI_TASK_CONFIG)

2. 修改run_evaluation.py #

添加自定义任务导入，指定任务并绑定构造函数：

import sys
import os
# 将自定义任务目录加入Python路径
sys.path.append(os.path.join(os.path.dirname(__file__), "evaluation"))

import lighteval
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.models.vllm.vllm_model import VLLMModelConfig
from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
from lighteval.utils.imports import is_accelerate_available
# 导入自定义任务
from custom_csi_task import custom_csi

if is_accelerate_available():
    from datetime import timedelta
    from accelerate import Accelerator, InitProcessGroupKwargs
    accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))])
else:
    accelerator = None

def main():
    evaluation_tracker = EvaluationTracker(
        output_dir="./results",
        save_details=True,
    )

    # 移除错误参数后的Pipeline配置
    pipeline_params = PipelineParameters(
        launcher_type=ParallelismManager.ACCELERATE,
    )

    model_config = VLLMModelConfig(
            model_name="HuggingFaceH4/zephyr-7b-beta",
            dtype="float16",
            use_chat_template=True,
    )

    # 指定自定义任务标识
    task = "custom|custom_csi"

    pipeline = Pipeline(
        tasks=task,  # 启用任务指定
        pipeline_parameters=pipeline_params,
        evaluation_tracker=evaluation_tracker,
        model_config=model_config,
        # 绑定自定义任务构造函数
        custom_tasks={"custom_csi": custom_csi}
    )

    pipeline.evaluate()
    pipeline.save_and_push_results()
    pipeline.show_results()

if __name__ == "__main__":
    main()

三、验证与运行 #

1. 确认数据集路径正确：CSI_TASK_CONFIG中的path需指向实际的csi_benchmark_advanced.jsonl文件
2. 安装依赖：确保已安装lighteval（建议安装最新版本：pip install git+https://github.com/huggingface/lighteval.git）、vllm、accelerate、numpy
3. 运行脚本：py .\run_evaluation.py

内容的提问来源于stack exchange，提问作者Mahmoud Hanouneh