{ "add_bos_token": false, "add_prefix_space": false, "added_tokens_decoder": { "100256": { "content": "<|pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100257": { "content": "<|end_of_text|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100258": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100259": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100260": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100261": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100262": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100263": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100264": { "content": "<|start_of_role|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100265": { "content": "<|end_of_role|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100266": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100267": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100268": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100269": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100270": { "content": "<image>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100271": { "content": "<caption>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100272": { "content": "</caption>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100273": { "content": "<footnote>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100274": { "content": "</footnote>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100275": { "content": "<formula>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100276": { "content": "</formula>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100277": { "content": "<list_item>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100278": { "content": "</list_item>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100279": { "content": "<page_footer>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100280": { "content": "</page_footer>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100281": { "content": "<page_header>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100282": { "content": "</page_header>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100283": { "content": "<picture>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100284": { "content": "</picture>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100285": { "content": "<section_header_level_1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100286": { "content": "</section_header_level_1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100287": { "content": "<section_header_level_2>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100288": { "content": "</section_header_level_2>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100289": { "content": "<section_header_level_3>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100290": { "content": "</section_header_level_3>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100291": { "content": "<section_header_level_4>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100292": { "content": "</section_header_level_4>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100293": { "content": "<section_header_level_5>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100294": { "content": "</section_header_level_5>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100295": { "content": "<section_header_level_6>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100296": { "content": "</section_header_level_6>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100297": { "content": "<otsl>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100298": { "content": "</otsl>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100299": { "content": "<checkbox_selected>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100300": { "content": "</checkbox_selected>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100301": { "content": "<checkbox_unselected>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100302": { "content": "</checkbox_unselected>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100303": { "content": "<form>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100304": { "content": "</form>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100305": { "content": "<key_value_region>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100306": { "content": "</key_value_region>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100307": { "content": "<key_", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100308": { "content": "</key_", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100309": { "content": "<value_", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100310": { "content": "</value_", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100311": { "content": "<link_", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100312": { "content": "<chart>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100313": { "content": "</chart>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100314": { "content": "<page_break>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100315": { "content": "<smiles>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100316": { "content": "</smiles>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100317": { "content": "</text>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100318": { "content": "<paragraph>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100319": { "content": "</paragraph>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100320": { "content": "<references>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100321": { "content": "</references>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100322": { "content": "<ordered_list>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100323": { "content": "</ordered_list>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100324": { "content": "<unordered_list>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100325": { "content": "</unordered_list>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100326": { "content": "<group>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100327": { "content": "<doctag>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100328": { "content": "</doctag>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100329": { "content": "<rec_", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100330": { "content": "<fcel>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100331": { "content": "<ecel>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100332": { "content": "<lcel>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100333": { "content": "<ucel>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100334": { "content": "<xcel>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100335": { "content": "<nl>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100336": { "content": "<ched>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100337": { "content": "<rhed>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100338": { "content": "<|unk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100339": { "content": "<fake_token_around_image>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100340": { "content": "<global-img>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100341": { "content": "<row_2_col_4>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100342": { "content": "<row_3_col_1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100343": { "content": "<row_3_col_2>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100344": { "content": "<row_3_col_3>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100345": { "content": "<row_3_col_4>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100346": { "content": "<row_4_col_1>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100347": { "content": "<row_4_col_2>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100348": { "content": "<row_4_col_3>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100349": { "content": "<row_4_col_4>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100350": { "content": "<code>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100351": { "content": "</code>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100352": { "content": "<end_of_utterance>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<fake_token_around_image>", "<image>", "<end_of_utterance>" ], "bos_token": "<|start_of_role|>", "clean_up_tokenization_spaces": false, "eos_token": "<|end_of_text|>", "errors": "replace", "extra_special_tokens": {}, "model_max_length": 8192, "pad_token": "<|end_of_text|>", "padding_side": "left", "processor_class": "Idefics3Processor", "tokenizer_class": "GPT2Tokenizer", "unk_token": "<|unk|>" }