Update tokenizer_config.json

#20
by Jiqing - opened
Files changed (1) hide show
  1. tokenizer_config.json +2 -10
tokenizer_config.json CHANGED
@@ -4930,14 +4930,6 @@
4930
  "rstrip": false,
4931
  "single_word": false,
4932
  "special": false
4933
- },
4934
- "100352": {
4935
- "content": "<pad>",
4936
- "lstrip": false,
4937
- "normalized": false,
4938
- "rstrip": false,
4939
- "single_word": false,
4940
- "special": true
4941
  }
4942
  },
4943
  "bos_token": null,
@@ -4946,12 +4938,12 @@
4946
  "eos_token": null,
4947
  "extra_special_tokens": {
4948
  "image_token": "<|img|>",
4949
- "pad_token": "<pad>"
4950
  },
4951
  "image_token": "<|img|>",
4952
  "legacy": true,
4953
  "model_max_length": 1000000000000000019884624838656,
4954
- "pad_token": "<pad>",
4955
  "processor_class": "AriaProcessor",
4956
  "sp_model_kwargs": {},
4957
  "spaces_between_special_tokens": false,
 
4930
  "rstrip": false,
4931
  "single_word": false,
4932
  "special": false
 
 
 
 
 
 
 
 
4933
  }
4934
  },
4935
  "bos_token": null,
 
4938
  "eos_token": null,
4939
  "extra_special_tokens": {
4940
  "image_token": "<|img|>",
4941
+ "pad_token": "<|endoftext|>"
4942
  },
4943
  "image_token": "<|img|>",
4944
  "legacy": true,
4945
  "model_max_length": 1000000000000000019884624838656,
4946
+ "pad_token": "<|endoftext|>",
4947
  "processor_class": "AriaProcessor",
4948
  "sp_model_kwargs": {},
4949
  "spaces_between_special_tokens": false,