Upload tokenizer
Browse files- chat_template.jinja +5 -0
- tokenizer_config.json +1 -1
chat_template.jinja
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
|
| 2 |
+
|
| 3 |
+
'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
|
| 4 |
+
|
| 5 |
+
' }}{% endif %}
|
tokenizer_config.json
CHANGED
|
@@ -2057,7 +2057,7 @@
|
|
| 2057 |
"input_ids",
|
| 2058 |
"attention_mask"
|
| 2059 |
],
|
| 2060 |
-
"model_max_length":
|
| 2061 |
"pad_token": "<|reserved_special_token_250|>",
|
| 2062 |
"padding_side": "left",
|
| 2063 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
|
|
|
| 2057 |
"input_ids",
|
| 2058 |
"attention_mask"
|
| 2059 |
],
|
| 2060 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 2061 |
"pad_token": "<|reserved_special_token_250|>",
|
| 2062 |
"padding_side": "left",
|
| 2063 |
"tokenizer_class": "PreTrainedTokenizerFast"
|