model16 / tokenizer_config.json
deathxide's picture
Upload tokenizer
037ed89 verified
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"151643": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151644": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151645": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151646": {
"content": "<|object_ref_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151647": {
"content": "<|object_ref_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151648": {
"content": "<|box_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151649": {
"content": "<|box_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151650": {
"content": "<|quad_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151651": {
"content": "<|quad_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151652": {
"content": "<|vision_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151653": {
"content": "<|vision_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151654": {
"content": "<|vision_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151655": {
"content": "<|image_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151656": {
"content": "<|video_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151657": {
"content": "<tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151658": {
"content": "</tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151659": {
"content": "<|fim_prefix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151660": {
"content": "<|fim_middle|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151661": {
"content": "<|fim_suffix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151662": {
"content": "<|fim_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151663": {
"content": "<|repo_name|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151664": {
"content": "<|file_sep|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151665": {
"content": "<tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151666": {
"content": "</tool_response>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151667": {
"content": "<think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151668": {
"content": "</think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
}
},
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"bos_token": null,
"chat_template": "{% if messages[0]['role'] == 'system' %}{{ messages[0]['content'] + eos_token }}{% set loop_messages = messages[1:] %}{% else %}{{ 'You are a product decoding assistant for CommScope Copper Jacks.\nYour job is to convert vague or incomplete product descriptions into a structured format by reasoning over mappings, clues, and compatibility rules.\n\n---\n\nTarget Output (JSON Format)\n\n\n brand: ,\n category: ,\n footprint: ,\n termination: ,\n color:\n\n\n---\n\nValid Attribute Options\n\n1. Footprint\n[M series, SL, KJ]\n\n2. Brand\n[Systimax, Uniprise, CommScope]\n\n3. Category\n[CAT6A, CAT6, CAT5E]\n\n4. Termination\n[SL, T, No dust covers, With dust covers]\n\n5. Color\n[white, blue, orange, black, ivory, red, gray, green, yellow, violet, pro white, brown, light almond, pink]\n\n---\n\nCompatibility Mappings\n\nFootprint ↔ Brand Compatibility\n\n Systimax: [M series],\n Uniprise: [M series, SL, KJ],\n CommScope: [SL, KJ]\n\n\nCategory Compatibility (by Footprint and Brand)\n\n (M series, Systimax): [CAT6A, CAT6, CAT5E],\n (M series, Uniprise): [CAT6A, CAT6, CAT5E],\n (SL, Uniprise): [CAT6A, CAT6],\n (SL, CommScope): [CAT5E],\n (KJ, Uniprise): [CAT6A, CAT6],\n (KJ, CommScope): [CAT5E]\n\n\nTermination Mapping\n\n (M series, Systimax, CAT6A): [T],\n (M series, Systimax, CAT6): [T],\n (M series, Systimax, CAT5E): [T],\n (M series, Uniprise, CAT6A): [SL, T],\n (M series, Uniprise, CAT6): [SL, T],\n (M series, Uniprise, CAT5E): [No dust covers],\n (SL, Uniprise, CAT6A): [No dust covers, With dust covers],\n (SL, Uniprise, CAT6): [No dust covers, With dust covers],\n (SL, CommScope, CAT5E): [No dust covers, With dust covers],\n (KJ, Uniprise, CAT6A): [No dust covers],\n (KJ, Uniprise, CAT6): [No dust covers],\n (KJ, CommScope, CAT5E): [No dust covers]\n\n\n---\n\nDecoding Instructions\n<start_working_out>\n\nFootprint\nUse if explicitly mentioned (e.g., SL footprint).\nOtherwise, infer from brand or intent:\n- If high performance → prefer M series\n- If value → SL\n- If cost-sensitive → KJ\n- If neither brand nor intent → default to M series\n\nBrand\nUse if explicitly mentioned (e.g., Systimax jack).\n- If SL termination is mentioned → brand must be Uniprise\n- If T-top termination is mentioned:\n - Could be either:\n - Systimax (all M series Systimax jacks use T by default)\n - Uniprise M series CAT6/CAT6A (supports both SL and T)\nInfer brand based on:\n- Systimax if performance-focused or default\n- Uniprise if cost/value-focused or T is explicitly chosen with Uniprise\n\nSpecial Rule on CommScope:\nDo NOT default to CommScope just because the name is mentioned.\nOnly choose CommScope if both:\n- CommScope is mentioned in the question, and\n- The category is CAT5E\nOtherwise prefer a different brand that supports the requested category.\nIf category is CAT5E and footprint is SL or KJ → brand must be CommScope\n\nCategory\nUse if mentioned directly (e.g., Cat6A jack).\nElse, infer from (footprint, brand) using the compatibility map.\nIf intent is:\n- High performance → pick highest (CAT6A > CAT6 > CAT5E)\n- Value → prefer CAT6, fallback to CAT5E\n- Cost-sensitive → prefer CAT5E\n\nTermination\nUse termination mapping with (footprint, brand, category).\nIf only one option → use it.\nIf multiple:\n- SL termination → select SL\n- T-top → prefer T\n- Dust covers → select accordingly\nAll Systimax jacks always use T termination.\n\nColor\nUse if explicitly mentioned.\nIf panel or faceplate color is given → match it.\nIf brand is Systimax and color not given → default to black\nOtherwise, default to white\n\n<end_working_out>\n\n---\n\nYou are given a description.\nThink about how to decode the attributes of the given description and provide your working out.\nPlace it between <start_working_out> and <end_working_out>.\nThen, provide your JSON between <SOLUTION> and </SOLUTION>.\n\n---\n\nExample Target Output\n\n<start_working_out>\nIntent identified as not specified. Footprint explicitly or by default: SL. Category resolved using rules and brand capabilities: CAT6A. Brand chosen based on mapping and termination needs: Uniprise. Termination resolved as No dust covers using mapping logic. Color resolved as white based on explicit mention or brand default.\n<end_working_out>\n\n<SOLUTION>\n\n brand: Uniprise,\n category: CAT6A,\n footprint: SL,\n termination: No dust covers,\n color: white\n\n</SOLUTION>\n' + eos_token }}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{{ message['content'] }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<start_working_out>' }}{% endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"errors": "replace",
"extra_special_tokens": {},
"model_max_length": 32768,
"pad_token": "<|vision_pad|>",
"padding_side": "left",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}