mistral-7b-v03-BoundingDocs-rephrased / special_tokens_map.json
simogiova's picture
Upload tokenizer
9c2bb4b verified
raw
history blame contribute delete
799 Bytes
{
"additional_special_tokens": [
"<|endanswer|>",
"<|pad|>",
"<|startdocument|>",
"<|enddocument|>",
"<|starttask|>",
"<|endtask|>",
"<|startanswer|>",
"<|startpos|>",
"<|endpos|>"
],
"bos_token": {
"content": "<|startdocument|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endanswer|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<|pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}