3장: lm-evaluation-harness 심층 분석

# 기본 설치
pip install lm-eval
 
# 특정 모델 백엔드 포함 설치
pip install lm-eval[vllm]
pip install lm-eval[api]  # OpenAI, Anthropic 등 API 백엔드
 
# 개발 모드 설치 (소스에서)
git clone https://github.com/EleutherAI/lm-evaluation-harness
cd lm-evaluation-harness
pip install -e ".[dev]"

# 단일 태스크 실행
lm_eval --model hf \
    --model_args pretrained=meta-llama/Llama-3.1-8B-Instruct \
    --tasks mmlu \
    --num_fewshot 5 \
    --batch_size 16 \
    --output_path results/
 
# 여러 태스크 동시 실행
lm_eval --model hf \
    --model_args pretrained=meta-llama/Llama-3.1-8B-Instruct \
    --tasks mmlu,hellaswag,arc_challenge,truthfulqa_mc2 \
    --batch_size auto \
    --output_path results/
 
# vLLM 백엔드 사용
lm_eval --model vllm \
    --model_args pretrained=meta-llama/Llama-3.1-8B-Instruct,tensor_parallel_size=2 \
    --tasks mmlu \
    --batch_size auto

import lm_eval
 
results = lm_eval.simple_evaluate(
    model="hf",
    model_args="pretrained=meta-llama/Llama-3.1-8B-Instruct",
    tasks=["mmlu", "hellaswag"],
    num_fewshot=5,
    batch_size=16,
    log_samples=True,
)
 
# 결과 확인
for task_name, task_result in results["results"].items():
    print(f"{task_name}: {task_result}")

# 태스크 메타데이터
task: my_custom_task
group: my_benchmark_group
dataset_path: my_org/my_dataset
dataset_name: default
test_split: test
validation_split: validation
 
# 프롬프트 구성
doc_to_text: "Question: {{question}}\nAnswer:"
doc_to_target: "{{answer}}"
 
# 평가 설정
output_type: generate
generation_kwargs:
  max_gen_toks: 128
  temperature: 0.0
  do_sample: false
 
# few-shot 설정
num_fewshot: 5
fewshot_split: validation
 
# 메트릭 정의
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
  - metric: f1
    aggregation: mean
    higher_is_better: true
 
# 메타데이터
metadata:
  version: 1.0

task: custom_multiple_choice
dataset_path: my_org/my_mc_dataset
test_split: test
 
output_type: loglikelihood
 
doc_to_text: |
  다음 질문에 대해 가장 적절한 답을 선택하세요.
 
  질문: {{question}}
  A) {{choices[0]}}
  B) {{choices[1]}}
  C) {{choices[2]}}
  D) {{choices[3]}}
  정답:
 
doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}"
doc_to_choice: ["A", "B", "C", "D"]
 
metric_list:
  - metric: acc
    aggregation: mean
    higher_is_better: true
  - metric: acc_norm
    aggregation: mean
    higher_is_better: true

mkdir -p lm_eval/tasks/korean_reasoning
touch lm_eval/tasks/korean_reasoning/_default_template.yaml
touch lm_eval/tasks/korean_reasoning/korean_commonsense.yaml

group: korean_reasoning
dataset_path: my_org/korean_commonsense_qa
output_type: loglikelihood
 
num_fewshot: 5
fewshot_split: validation
 
metadata:
  version: 1.0

include: _default_template.yaml
task: korean_commonsense_qa
 
dataset_name: commonsense
test_split: test
validation_split: validation
 
doc_to_text: !function utils.format_question
doc_to_target: "{{answer}}"
doc_to_choice: "{{choices}}"
 
metric_list:
  - metric: acc
    aggregation: mean
    higher_is_better: true
 
filter_list:
  - name: remove_whitespace
    filter:
      - function: remove_whitespace

def format_question(doc: dict) -> str:
    """데이터셋 문서를 프롬프트 형식으로 변환합니다."""
    question = doc["question"]
    choices = doc["choices"]
    
    prompt = f"질문: {question}\n"
    for i, choice in enumerate(choices):
        label = chr(ord("A") + i)
        prompt += f"{label}) {choice}\n"
    prompt += "정답:"
    
    return prompt

lm_eval --model hf \
    --model_args pretrained=meta-llama/Llama-3.1-8B-Instruct \
    --tasks korean_commonsense_qa \
    --num_fewshot 5 \
    --include_path ./lm_eval/tasks/korean_reasoning \
    --output_path results/korean/

group: my_evaluation_suite
task:
  - mmlu
  - hellaswag
  - arc_challenge
  - custom_korean_qa
aggregate_metric_list:
  - metric: acc
    aggregation: mean
    weight_by_size: true

# 그룹 단위로 실행
lm_eval --model hf \
    --model_args pretrained=my-model \
    --tasks my_evaluation_suite

filter_list:
  - name: extraction_pipeline
    filter:
      - function: regex
        regex_pattern: "답: ([A-D])"
        group_select: 1
      - function: map
        mapping:
          A: 0
          B: 1
          C: 2
          D: 3