ContraCode-Roberta / added_tokens.json
Ensheng's picture
20221117070653/checkpoints-143000-0.0004
e8ddc87
{"statement_identifier": 50265, "type_identifier": 50266, "int_literal": 50267, "regex_flags": 50268, "namespace": 50269, "keyword": 50270, "identifier": 50271, "escape_sequence": 50272, "class_variable": 50273, "field_identifier": 50274, "shorthand_property_identifier_pattern": 50275, "simple_symbol": 50276, "boolean": 50277, "constant": 50278, "heredoc_end": 50279, "global_variable": 50280, "php_tag": 50281, "decimal_floating_point_literal": 50282, "boolean_type": 50283, "separators": 50284, "void_type": 50285, "shorthand_property_identifier": 50286, "string_literal": 50287, "heredoc_content": 50288, "decimal_integer_literal": 50289, "extends": 50290, "instance_variable": 50291, "property_identifier": 50292, "regex_pattern": 50293, "string_fragment": 50294, "character_literal": 50295, "hash_key_symbol": 50296, "float_literal": 50297, "raw_string_literal": 50298, "rune_literal": 50299, "package_identifier": 50300, "null_literal": 50301, "string_content": 50302, "label_name": 50303, "hex_integer_literal": 50304, "heredoc_beginning": 50305}