BPE-Multilingual-72k / special_tokens_map.json
fhai50032's picture
Upload tokenizer
c367a28 verified
raw
history blame contribute delete
341 Bytes
{
"additional_special_tokens": [
"<|im_start|>user",
"<|im_start|>assistant",
"<|im_start|>system",
"<|tool_start|>",
"<|tool_end|>",
"<|im_end|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>"
],
"bos_token": "<|begin_of_text|>",
"eos_token": "<|end_of_text|>",
"pad_token": "<|pad|>"
}