models.yaml

# This is a config file for the models available in HarmBench. The entries in this config specify model names, how to load the models, how many GPUs
# to request for the models (e.g., if using SLURM), the model type, and other information.

# ===================== Documentation for the fields ===================== #
# Required fields for all models::
#     model_type: The type of model. This can be "open_source", "closed_source", "open_source_multimodal", or "closed_source_multimodal".
#                 All models are required to have this field in their config. This is used to determine which other fields to expect.

# Required fields for open_source models::
#     model: A dictionary containing arguments to load_model_and_tokenizer in `HarmBench/baselines/model_utils.py`. This is used to load the model
#            and tokenizer. This dictionary is required to have the `model_name_or_path` field. Other fields are optional and can be used to
#            override the default arguments to load_model_and_tokenizer.
#     num_gpus: The number of GPUs to request for the model. This is used to determine the number of GPUs to request when using SLURM.
#               Some methods also load open source LLMs using vLLM or create mini-clusters using ray. For these methods, we use `num_gpus`
#               to allocate the right number of GPUs for the model.
#               NOTE: The default values were chosen to work with 80GB A100 GPUs. If you are using different GPUs, you may need to adjust `num_gpus`.
# 
# Required fields for closed_source models::
#     model: A dictionary containing arguments to load_model_and_tokenizer in `model_utils.py`. This is used to load the model and tokenizer.
#            This dictionary is required to have the following fields, which are handled in `HarmBench/api_models.py`:
#            - model_name_or_path: The name of the API. This is used to select the API class in `HarmBench/api_models.py`.
#            - token: The token for accessing the model
#
# Required fields for open_source_multimodal models::
#     model: A dictionary containing arguments to load_model_and_tokenizer in `HarmBench/baselines/model_utils.py`. This is used to load the model and tokenizer.
# 
# Required fields for closed_source_multimodal models::
#     model: A dictionary containing arguments to load_model_and_tokenizer in `model_utils.py`. This is used to load the model and tokenizer.


# ===================== Open-Source Models ===================== #

# Llama 2
llama2_7b:
  model:
    model_name_or_path: meta-llama/Llama-2-7b-chat-hf
    use_fast_tokenizer: False
    dtype: float16
    chat_template: llama-2
  num_gpus: 1
  model_type: open_source

llama2_13b:
  model:
    model_name_or_path: meta-llama/Llama-2-13b-chat-hf
    use_fast_tokenizer: False
    dtype: float16
    chat_template: llama-2
  num_gpus: 1
  model_type: open_source

llama2_70b:
  model:
    model_name_or_path: meta-llama/Llama-2-70b-chat-hf
    use_fast_tokenizer: False
    dtype: float16
    chat_template: llama-2
  num_gpus: 2
  model_type: open_source

# Vicuna
vicuna_7b_v1_5:
  model:
    model_name_or_path: lmsys/vicuna-7b-v1.5
    use_fast_tokenizer: False
    dtype: float16
    chat_template: vicuna
  num_gpus: 1
  model_type: open_source

vicuna_13b_v1_5:
  model:
    model_name_or_path: lmsys/vicuna-13b-v1.5
    use_fast_tokenizer: False
    dtype: float16
    chat_template: vicuna
  num_gpus: 1
  model_type: open_source

# Koala
koala_7b:
  model:
    model_name_or_path: TheBloke/koala-7B-HF
    use_fast_tokenizer: False
    dtype: bfloat16
    chat_template: koala
  num_gpus: 1
  model_type: open_source

koala_13b:
  model:
    model_name_or_path: TheBloke/koala-13B-HF
    use_fast_tokenizer: False
    dtype: bfloat16
    chat_template: koala
  num_gpus: 1
  model_type: open_source

# Orca
orca_2_7b:
  model: 
    model_name_or_path: microsoft/Orca-2-7b
    use_fast_tokenizer: False
    dtype: bfloat16
    chat_template: orca-2
  num_gpus: 1
  model_type: open_source

orca_2_13b:
  model: 
    model_name_or_path: microsoft/Orca-2-13b
    use_fast_tokenizer: False
    dtype: bfloat16
    chat_template: orca-2
  num_gpus: 1
  model_type: open_source

# SOLAR
solar_10_7b_instruct:
  model:
    model_name_or_path: upstage/SOLAR-10.7B-Instruct-v1.0
    use_fast_tokenizer: False
    dtype: float16
  num_gpus: 1
  model_type: open_source

# OpenChat
openchat_3_5_1210:
  model:
    model_name_or_path: openchat/openchat-3.5-1210
    use_fast_tokenizer: False
    dtype: bfloat16
  num_gpus: 1
  model_type: open_source

# Starling
starling_7b:
  model: 
    model_name_or_path: berkeley-nest/Starling-LM-7B-alpha
    use_fast_tokenizer: False
    dtype: bfloat16
  num_gpus: 1
  model_type: open_source

# Mistral
mistral_7b_v2:
  model: 
    model_name_or_path: mistralai/Mistral-7B-Instruct-v0.2
    use_fast_tokenizer: False
    dtype: bfloat16
    chat_template: mistral
  num_gpus: 1
  model_type: open_source

mixtral_8x7b:
  model:
    model_name_or_path: mistralai/Mixtral-8x7B-Instruct-v0.1
    use_fast_tokenizer: False
    dtype: bfloat16
    chat_template: mistral
  num_gpus: 2
  model_type: open_source

# Zephyr
zephyr_7b:
  model:
    model_name_or_path: HuggingFaceH4/zephyr-7b-beta
    use_fast_tokenizer: False
    dtype: bfloat16
  num_gpus: 1
  model_type: open_source

# Zephyr 7B + R2D2 (adversarial training against GCG)
zephyr_7b_robust:
  model:
    model_name_or_path: cais/zephyr_7b_r2d2
    use_fast_tokenizer: False
    dtype: bfloat16
    chat_template: zephyr_7b_robust
  num_gpus: 1
  model_type: open_source

# Baichuan 2
baichuan2_7b:
  model:
    model_name_or_path: baichuan-inc/Baichuan2-7B-Chat
    use_fast_tokenizer: False
    dtype: bfloat16
    trust_remote_code: True
    chat_template: baichuan2
  num_gpus: 1
  model_type: open_source

baichuan2_13b:
  model:
    model_name_or_path: baichuan-inc/Baichuan2-13B-Chat
    use_fast_tokenizer: False
    dtype: bfloat16
    trust_remote_code: True
    chat_template: baichuan2
  num_gpus: 1
  model_type: open_source

# Qwen
qwen_7b_chat:
  model:
    model_name_or_path: Qwen/Qwen-7B-Chat
    use_fast_tokenizer: True
    dtype: bfloat16
    pad_token: <|extra_0|>
    eos_token: <|im_end|>
    trust_remote_code: True
    chat_template: qwen
  num_gpus: 1
  model_type: open_source

qwen_14b_chat:
  model:
    model_name_or_path: Qwen/Qwen-14B-Chat
    use_fast_tokenizer: True
    trust_remote_code: True
    dtype: bfloat16
    pad_token: <|extra_0|>
    eos_token: <|im_end|>
    chat_template: qwen
  num_gpus: 1
  model_type: open_source

qwen_72b_chat:
  model:
    model_name_or_path: Qwen/Qwen-72B-Chat
    use_fast_tokenizer: True
    dtype: bfloat16
    pad_token: <|extra_0|>
    eos_token: <|im_end|>
    trust_remote_code: True
    chat_template: qwen
  num_gpus: 4
  model_type: open_source



# ===================== Closed-Source Models ===================== #

# TODO: remove tokens before release
gpt-3.5-turbo-1106:
  model:
    model_name_or_path: gpt-3.5-turbo-1106
    token: <your_openai_token>
  model_type: closed_source

gpt-3.5-turbo-0613:
  model:
    model_name_or_path: gpt-3.5-turbo-0613
    token: <your_openai_token>
  model_type: closed_source

gpt-4-1106-preview:
  model:
    model_name_or_path: gpt-4-1106-preview
    token: <your_openai_token>
  model_type: closed_source
  
gpt-4-0613:
  model:
    model_name_or_path: gpt-4-0613
    token: <your_openai_token>
  model_type: closed_source

gpt-4-vision-preview:
  model:
    model_name_or_path: gpt-4-vision-preview
    token: <your_openai_token>
  model_type: closed_source


claude-instant-1:
  model:
    model_name_or_path: claude-instant-1
    token: <your_anthropic_token>
  model_type: closed_source
  
claude-2.1:
  model:
    model_name_or_path: claude-2.1
    token: <your_anthropic_token>
  model_type: closed_source
  
claude-2:
  model:
    model_name_or_path: claude-2
    token: <your_anthropic_token>
  model_type: closed_source

gemini:
  model:
    model_name_or_path: gemini-pro
    token: <your_google_ai_studio_token>
  model_type: closed_source

mistral-medium:
  mode:
    model_name_or_path: mistral-medium
    token: <your_mistral_token>
  model_type: closed_source


# ===================== Open-Source Multimodal Models ===================== #

llava_v1_5:
  model:
    model_name_or_path: LLaVA_v1_5
  model_type: open_source_multimodal
  num_gpus: 1

instructblip:
  model:
    model_name_or_path: InstructBLIP
  model_type: open_source_multimodal
  num_gpus: 1

qwen_vl_chat:
  model:
    model_name_or_path: Qwen_VL_Chat
  model_type: open_source_multimodal
  num_gpus: 1


# ===================== Closed-Source Multimodal Models ===================== #

gpt4v:
  model:
    model_name_or_path: GPT4V
  model_type: closed_source_multimodal