| {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}, "additional_special_tokens": ["statement_identifier", "type_identifier", "int_literal", "regex_flags", "namespace", "keyword", "identifier", "escape_sequence", "class_variable", "field_identifier", "shorthand_property_identifier_pattern", "operator", "ERROR", "simple_symbol", "boolean", "constant", "heredoc_end", "global_variable", "php_tag", "decimal_floating_point_literal", "boolean_type", "\"", "separators", "void_type", "shorthand_property_identifier", "string_literal", "number", "heredoc_content", "decimal_integer_literal", "extends", "instance_variable", "property_identifier", "regex_pattern", "comment", "integer", "string_fragment", "character_literal", "hash_key_symbol", "float_literal", "raw_string_literal", "rune_literal", "package_identifier", "null_literal", "text", "class", "string_content", "label_name", "hex_integer_literal", "heredoc_beginning", "string", "name", "none"]} |