# Evaluation Configuration
eval:
  metric: "pass@1" # only pass@1 is supported now
  num_tests_per_prompt: 1 # every prompt will be tested num_tests_per_prompt times and use the average score as the final score
  seed: 42

generation:
  backend: "vllm" # only vllm is supported for evaluation
  max_new_tokens: ${generation.vllm_cfg.max_model_len}
  temperature: 0.0
  top_p: 1.0
  top_k: -1 # -1 means disable
  num_prompts_per_step: -1 # -1 means pass all prompts at once
  model_name: "Qwen/Qwen2.5-Math-1.5B-Instruct"
  stop_token_ids: null
  stop_strings: null
  vllm_cfg:
    precision: "bfloat16"
    tensor_parallel_size: 1
    gpu_memory_utilization: 0.9
    max_model_len: 2048

tokenizer:
  name: ${generation.model_name} ## specify if you'd like to use a tokenizer different from the model's default
  chat_template: "default"

data:
  max_input_seq_length: ${generation.vllm_cfg.max_model_len} # useless since we directly use prompts in evaluation
  prompt_file: null
  system_prompt_file: null
  dataset_name: "HuggingFaceH4/aime_2024"
  dataset_key: "train"
  problem_key: "problem"
  solution_key: "answer"

env:
  math:
    num_workers: 8

cluster:
  gpus_per_node: 1
  num_nodes: 1
