Update tokenizer_config.json
#20
by
Jiqing
- opened
- tokenizer_config.json +2 -10
tokenizer_config.json
CHANGED
@@ -4930,14 +4930,6 @@
|
|
4930 |
"rstrip": false,
|
4931 |
"single_word": false,
|
4932 |
"special": false
|
4933 |
-
},
|
4934 |
-
"100352": {
|
4935 |
-
"content": "<pad>",
|
4936 |
-
"lstrip": false,
|
4937 |
-
"normalized": false,
|
4938 |
-
"rstrip": false,
|
4939 |
-
"single_word": false,
|
4940 |
-
"special": true
|
4941 |
}
|
4942 |
},
|
4943 |
"bos_token": null,
|
@@ -4946,12 +4938,12 @@
|
|
4946 |
"eos_token": null,
|
4947 |
"extra_special_tokens": {
|
4948 |
"image_token": "<|img|>",
|
4949 |
-
"pad_token": "
|
4950 |
},
|
4951 |
"image_token": "<|img|>",
|
4952 |
"legacy": true,
|
4953 |
"model_max_length": 1000000000000000019884624838656,
|
4954 |
-
"pad_token": "
|
4955 |
"processor_class": "AriaProcessor",
|
4956 |
"sp_model_kwargs": {},
|
4957 |
"spaces_between_special_tokens": false,
|
|
|
4930 |
"rstrip": false,
|
4931 |
"single_word": false,
|
4932 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4933 |
}
|
4934 |
},
|
4935 |
"bos_token": null,
|
|
|
4938 |
"eos_token": null,
|
4939 |
"extra_special_tokens": {
|
4940 |
"image_token": "<|img|>",
|
4941 |
+
"pad_token": "<|endoftext|>"
|
4942 |
},
|
4943 |
"image_token": "<|img|>",
|
4944 |
"legacy": true,
|
4945 |
"model_max_length": 1000000000000000019884624838656,
|
4946 |
+
"pad_token": "<|endoftext|>",
|
4947 |
"processor_class": "AriaProcessor",
|
4948 |
"sp_model_kwargs": {},
|
4949 |
"spaces_between_special_tokens": false,
|