Upload tokenizer

chat_template.jinja

@@ -1,25 +1,23 @@
-{{ bos_token}}{% if  messages[0]['role']==system%}
-{{ raise_exception('System message is not supported in gemma, it would be good to merget the system prompt with first user message')}}
+{{ bos_token }}{% if messages[0].role == 'system' %}
+{{ raise_exception('System message is not supported in gemma; merge the system prompt with the first user message') }}
 {% endif %}
-{% if not tools%}
-{{ raise_exception('You should provide tools, this model only trained for tool calling with Reasoning')}}
+{% if not tools %}
+{{ raise_exception('You should provide tools; this model was only trained for tool calling with reasoning') }}
 {% endif %}
-{% set first_message=True %}
+{% set first_message = True %}
 {% for message in messages %}
-{% if first_message and tools%}
-{{"<start_of_turn>human
-You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags.You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions.Here are the available tools:<tools> " ~ (tools | tojson) ~ " </tools>Use the following pydantic model json schema for each tool call you will make: {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
+  {% if first_message and tools %}
+{% raw -%}
+<start_of_turn>human
+You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags.You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions.Here are the available tools:<tools> {% endraw %}{{ tools }}{% raw -%} </tools>Use the following pydantic model json schema for each tool call you will make: {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
 <tool_call>
 {tool_call}
 </tool_call>Also, before making a call to a function take the time to plan the function to take. Make that thinking process between <think>{your thought}</think>
 
-" + message['content'] }}
-{% set first_message = False %}
-{% else %}
-{{'<start_of_turn>' + message['role'] + '
-' + message['content'] | trim + '<eos_turn><eos>
-'}}
-{% endif %}
+{% endraw %}{{ message.content }}{% raw -%}
+<eos_turn><eos>
+<start_of_turn>model
+{% endraw -%}
+    {% set first_message = False %}
+  {% endif %}
 {% endfor %}
-{% if add_generation_prompt %}{{'<start_of_turn>model
-'}}{% endif %}"

special_tokens_map.json

@@ -16,6 +16,18 @@
     "rstrip": false,
     "single_word": false
   },
-  "eos_token": "<eos>",
-  "pad_token": "<pad>"
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }