| { |
| "version": "1.0", |
| "truncation": null, |
| "padding": null, |
| "added_tokens": [ |
| { |
| "id": 0, |
| "content": "<|endoftext|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 1, |
| "content": "<|psep|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 2, |
| "content": "</|psep|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 3, |
| "content": "<|vsep|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 4, |
| "content": "<|bsep|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 5, |
| "content": "</|bsep|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 6, |
| "content": "<|pad|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 7, |
| "content": "<|meter_0|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 8, |
| "content": "<|meter_1|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 9, |
| "content": "<|meter_2|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 10, |
| "content": "<|meter_3|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 11, |
| "content": "<|meter_4|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 12, |
| "content": "<|meter_5|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 13, |
| "content": "<|meter_6|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 14, |
| "content": "<|meter_7|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 15, |
| "content": "<|meter_8|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 16, |
| "content": "<|meter_9|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 17, |
| "content": "<|meter_10|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 18, |
| "content": "<|meter_11|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 19, |
| "content": "<|meter_12|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 20, |
| "content": "<|meter_13|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 21, |
| "content": "<|meter_14|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 22, |
| "content": "<|meter_15|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 23, |
| "content": "<|meter_16|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 24, |
| "content": "<|theme_0|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 25, |
| "content": "<|theme_1|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 26, |
| "content": "<|theme_2|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 27, |
| "content": "<|theme_3|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 28, |
| "content": "<|theme_4|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 29, |
| "content": "<|theme_5|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 30, |
| "content": "<|theme_6|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 31, |
| "content": "<|theme_7|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 32, |
| "content": "<|theme_8|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 33, |
| "content": "<|theme_9|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 34, |
| "content": "<|theme_10|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 35, |
| "content": "<|theme_11|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 36, |
| "content": "<|theme_12|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 37, |
| "content": "<|theme_13|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 38, |
| "content": "<|theme_14|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 39, |
| "content": "<|theme_15|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 40, |
| "content": "<|theme_16|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 41, |
| "content": "<|theme_17|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 42, |
| "content": "<|theme_18|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 43, |
| "content": "<|res_0|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 44, |
| "content": "<|res_1|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 45, |
| "content": "<|res_2|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 46, |
| "content": "<|res_3|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 47, |
| "content": "<|res_4|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 48, |
| "content": "<|res_5|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 49, |
| "content": "<|res_6|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 50, |
| "content": "<|res_7|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 51, |
| "content": "<|res_8|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 52, |
| "content": "<|res_9|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| } |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "UnicodeScripts" |
| }, |
| "post_processor": null, |
| "decoder": null, |
| "model": { |
| "type": "BPE", |
| "dropout": null, |
| "unk_token": null, |
| "continuing_subword_prefix": null, |
| "end_of_word_suffix": null, |
| "fuse_unk": false, |
| "byte_fallback": false, |
| "vocab": { |
| "<|endoftext|>": 0, |
| "<|psep|>": 1, |
| "</|psep|>": 2, |
| "<|vsep|>": 3, |
| "<|bsep|>": 4, |
| "</|bsep|>": 5, |
| "<|pad|>": 6, |
| "<|meter_0|>": 7, |
| "<|meter_1|>": 8, |
| "<|meter_2|>": 9, |
| "<|meter_3|>": 10, |
| "<|meter_4|>": 11, |
| "<|meter_5|>": 12, |
| "<|meter_6|>": 13, |
| "<|meter_7|>": 14, |
| "<|meter_8|>": 15, |
| "<|meter_9|>": 16, |
| "<|meter_10|>": 17, |
| "<|meter_11|>": 18, |
| "<|meter_12|>": 19, |
| "<|meter_13|>": 20, |
| "<|meter_14|>": 21, |
| "<|meter_15|>": 22, |
| "<|meter_16|>": 23, |
| "<|theme_0|>": 24, |
| "<|theme_1|>": 25, |
| "<|theme_2|>": 26, |
| "<|theme_3|>": 27, |
| "<|theme_4|>": 28, |
| "<|theme_5|>": 29, |
| "<|theme_6|>": 30, |
| "<|theme_7|>": 31, |
| "<|theme_8|>": 32, |
| "<|theme_9|>": 33, |
| "<|theme_10|>": 34, |
| "<|theme_11|>": 35, |
| "<|theme_12|>": 36, |
| "<|theme_13|>": 37, |
| "<|theme_14|>": 38, |
| "<|theme_15|>": 39, |
| "<|theme_16|>": 40, |
| "<|theme_17|>": 41, |
| "<|theme_18|>": 42, |
| "<|res_0|>": 43, |
| "<|res_1|>": 44, |
| "<|res_2|>": 45, |
| "<|res_3|>": 46, |
| "<|res_4|>": 47, |
| "<|res_5|>": 48, |
| "<|res_6|>": 49, |
| "<|res_7|>": 50, |
| "<|res_8|>": 51, |
| "<|res_9|>": 52, |
| " ": 53, |
| "/": 54, |
| "0": 55, |
| "1": 56, |
| "2": 57, |
| "3": 58, |
| "4": 59, |
| "5": 60, |
| "6": 61, |
| "7": 62, |
| "8": 63, |
| "9": 64, |
| "<": 65, |
| ">": 66, |
| "_": 67, |
| "b": 68, |
| "e": 69, |
| "h": 70, |
| "m": 71, |
| "p": 72, |
| "r": 73, |
| "s": 74, |
| "t": 75, |
| "v": 76, |
| "|": 77, |
| "~": 78, |
| "ء": 79, |
| "أ": 80, |
| "ؤ": 81, |
| "ئ": 82, |
| "ا": 83, |
| "ب": 84, |
| "ة": 85, |
| "ت": 86, |
| "ث": 87, |
| "ج": 88, |
| "ح": 89, |
| "خ": 90, |
| "د": 91, |
| "ذ": 92, |
| "ر": 93, |
| "ز": 94, |
| "س": 95, |
| "ش": 96, |
| "ص": 97, |
| "ض": 98, |
| "ط": 99, |
| "ظ": 100, |
| "ع": 101, |
| "غ": 102, |
| "ف": 103, |
| "ق": 104, |
| "ك": 105, |
| "ل": 106, |
| "م": 107, |
| "ن": 108, |
| "ه": 109, |
| "و": 110, |
| "ى": 111, |
| "ي": 112, |
| "ً": 113, |
| "ٌ": 114, |
| "ٍ": 115, |
| "َ": 116, |
| "ُ": 117, |
| "ِ": 118, |
| "ّ": 119, |
| "ْ": 120 |
| }, |
| "merges": [] |
| } |
| } |
