{ "add_bos_token": false, "add_prefix_space": false, "added_tokens_decoder": { "0": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<|pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "<|unk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "<|sep|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "<|mask|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "<|cls|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "6": { "content": "<|image|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "7": { "content": "<|audio|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "8": { "content": "<|user|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "9": { "content": "<|system|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "10": { "content": "<|assistant|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "11": { "content": "<|extra_id_0|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "12": { "content": "<|extra_id_1|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "13": { "content": "<|extra_id_2|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "14": { "content": "<|extra_id_3|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "15": { "content": "<|extra_id_4|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16": { "content": "<|extra_id_5|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "17": { "content": "<|extra_id_6|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "18": { "content": "<|extra_id_7|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "19": { "content": "<|extra_id_8|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "20": { "content": "<|extra_id_9|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "21": { "content": "<|extra_id_10|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "22": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "23": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "24": { "content": "<|extra_id_13|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "25": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "26": { "content": "<|im_sep|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "27": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "28": { "content": "<|resident_reg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "29": { "content": "<|foreigner_reg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "30": { "content": "<|business_reg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "31": { "content": "<|credit_card|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32": { "content": "<|passport|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "33": { "content": "<|driver_license|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "34": { "content": "<|telephone|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "35": { "content": "<|health_insurance|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "36": { "content": "<|bank_account|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "37": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "38": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "39": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "40": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false } }, "additional_special_tokens": [ "<|endoftext|>", "<|pad|>", "<|unk|>", "<|sep|>", "<|mask|>", "<|cls|>", "<|image|>", "<|audio|>", "<|user|>", "<|system|>", "<|assistant|>", "<|extra_id_0|>", "<|extra_id_1|>", "<|extra_id_2|>", "<|extra_id_3|>", "<|extra_id_4|>", "<|extra_id_5|>", "<|extra_id_6|>", "<|extra_id_7|>", "<|extra_id_8|>", "<|extra_id_9|>", "<|extra_id_10|>", "<|extra_id_13|>", "<|im_start|>", "<|im_sep|>", "<|im_end|>", "<|resident_reg|>", "<|foreigner_reg|>", "<|business_reg|>", "<|credit_card|>", "<|passport|>", "<|driver_license|>", "<|telephone|>", "<|health_insurance|>", "<|bank_account|>" ], "bos_token": "<|endoftext|>", "clean_up_tokenization_spaces": true, "cls_token": "<|cls|>", "eod_token": "<|endoftext|>", "eos_token": "<|im_end|>", "errors": "replace", "extra_special_tokens": {}, "mask_token": "<|mask|>", "max_length": 7680, "model_max_length": 32768, "pad_token": "<|pad|>", "sep_token": "<|sep|>", "tokenizer_class": "GPT2Tokenizer", "unk_token": "<|unk|>", "vocab_size": 102400 }