{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 152, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 153, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 154, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 155, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 156, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 157, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 158, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 159, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 160, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 161, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 162, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 163, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 164, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 165, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 166, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 167, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 168, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 169, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 170, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 171, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 172, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 173, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 174, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 175, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 176, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 177, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 178, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 179, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 180, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 181, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 182, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 183, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 184, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 185, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 186, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 187, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 188, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 189, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 190, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 191, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 192, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 193, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 194, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 195, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 196, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 197, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 198, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 199, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 200, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 201, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 202, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 203, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 204, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 205, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 206, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 207, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 208, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 209, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 210, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 211, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 212, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 213, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 214, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 215, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 216, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 217, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 218, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 219, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 220, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 221, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 222, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 223, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 224, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 225, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 226, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 227, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 228, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 229, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 230, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 231, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 232, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 233, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 234, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 235, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 236, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 237, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 238, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 239, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 240, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 241, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 242, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 243, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 244, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 245, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 246, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 247, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 248, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 249, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 250, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 251, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "Nmt" }, { "type": "NFKC" }, { "type": "Replace", "pattern": { "Regex": " {2,}" }, "content": " " } ] }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Metaspace", "replacement": "▁", "add_prefix_space": true }, { "type": "Digits", "individual_digits": true }, { "type": "Punctuation", "behavior": "Isolated" } ] }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": { "": { "id": "", "ids": [ 1 ], "tokens": [ "" ] } } }, "decoder": { "type": "Metaspace", "replacement": "▁", "add_prefix_space": true }, "model": { "type": "Unigram", "unk_id": 2, "vocab": [ [ "", 0.0 ], [ "", 0.0 ], [ "", 0.0 ], [ "c", -1.8325501267458328 ], [ "C", -2.085833731170693 ], [ ")", -2.1794139187786303 ], [ "(", -2.1794139187786303 ], [ "1", -2.6473270555708233 ], [ "O", -2.908316332235933 ], [ "2", -3.007127858046692 ], [ "=", -3.138519245261141 ], [ "N", -3.3313758037317305 ], [ "3", -3.830167355077707 ], [ "]", -4.021033373135008 ], [ "[", -4.021033373135008 ], [ "n", -4.277442252892587 ], [ "ccc", -4.4391790758440415 ], [ "H", -4.4851081400686414 ], [ "+", -4.52751730236701 ], [ "-", -4.544507994020134 ], [ "CC", -4.607523195934234 ], [ "cccc", -4.666119693433614 ], [ "F", -4.670620867963353 ], [ "4", -5.121096337428546 ], [ "▁C", -5.163538144660347 ], [ "▁CC", -5.186433194987645 ], [ "S", -5.2409230340286985 ], [ "cc", -5.297230130780081 ], [ "CCC", -5.398382240870886 ], [ "▁", -5.479601908139957 ], [ "Cl", -5.516677182880245 ], [ "NC", -5.772079862146344 ], [ "5", -6.019033955686507 ], [ "CCCC", -6.091218058928987 ], [ "Br", -6.096851374684951 ], [ "▁CO", -6.1226121436428524 ], [ "nc", -6.189770897333768 ], [ "OC", -6.253272290547393 ], [ "#", -6.259707039207268 ], [ "l", -6.41791255462504 ], [ "▁O", -6.467668724107586 ], [ "s", -6.547807068455772 ], [ "▁CCC", -6.602039316796626 ], [ "CCN", -6.674074883113083 ], [ "o", -6.6921881645660575 ], [ "nnc", -6.971819763184997 ], [ "nn", -7.0872976248625665 ], [ "CCCCC", -7.19300639125257 ], [ "no", -7.276252813956157 ], [ "▁COC", -7.329033247669198 ], [ "NCC", -7.3764529490665005 ], [ "CCOCC", -7.458908285977033 ], [ "ncn", -7.4840036438888 ], [ "▁N", -7.486276057964793 ], [ "▁CCOC", -7.499290910945031 ], [ "▁CN", -7.513425929350326 ], [ "OCC", -7.516116036082575 ], [ "sc", -7.583828971249213 ], [ "cnc", -7.58506199500868 ], [ "ncc", -7.634890007798665 ], [ "6", -7.694663300272712 ], [ "i", -7.694663300272712 ], [ "oc", -7.72564267417347 ], [ "ccnc", -7.762658264172304 ], [ "CO", -7.8222879947709405 ], [ "CCCN", -7.871725413837872 ], [ "cccn", -7.875369198920738 ], [ "▁CCCC", -7.9568764188470436 ], [ "SC", -8.002337294122125 ], [ "COC", -8.010403671108064 ], [ "▁CCO", -8.015421132661498 ], [ "▁CS", -8.018832409844851 ], [ "SCC", -8.022823157712045 ], [ "CN", -8.039346171655314 ], [ "cn", -8.070918475660292 ], [ "CCS", -8.112916095350831 ], [ "csc", -8.148597619230532 ], [ "CS", -8.161082004581061 ], [ "cnn", -8.163475617837227 ], [ "ccs", -8.186959804241676 ], [ "cco", -8.198048368242024 ], [ "▁CCN", -8.217144700567495 ], [ "P", -8.305247568946438 ], [ "ccn", -8.322699921731036 ], [ "CCOC", -8.345115301049947 ], [ "▁COCC", -8.355816885941763 ], [ "CNC", -8.372424615766462 ], [ "CCO", -8.441677846224742 ], [ "▁CCCCC", -8.571159375939569 ], [ "cs", -8.66327559789252 ], [ "I", -8.684215822966232 ], [ "▁CCCN", -8.774610381897997 ], [ "7", -8.850882489632898 ], [ "▁CCS", -8.89157844887687 ], [ "CCCO", -8.925245618002188 ], [ "CCCCCC", -9.014693669780256 ], [ "cncc", -9.032140774783397 ], [ "OCCOCCOCCOCCOCC", -9.07904936490989 ], [ "OCCO", -9.119198365882852 ], [ "CSC", -9.15725634612342 ], [ "CCCCCCCCC", -9.222531716633128 ], [ "nccc", -9.222653573940203 ], [ "nnn", -9.248991350115892 ], [ "on", -9.276583689709438 ], [ "8", -9.3008824896329 ], [ "▁CCCCCCC", -9.376036782967104 ], [ "▁Cl", -9.37890788780132 ], [ "scc", -9.42683337318626 ], [ "OCCOC", -9.57046662697634 ], [ "COCCOCCOCCOCCOCC", -9.585839446612466 ], [ "B", -9.600822251371683 ], [ "CSCC", -9.607047371907186 ], [ "COCC", -9.630746851225965 ], [ "e", -9.634215822966231 ], [ "snn", -9.714126912203849 ], [ "CCCCCCCCCCC", -9.717323815455336 ], [ "▁NN", -9.723204995229974 ], [ "▁CCCCCC", -9.825838349839016 ], [ "▁CCCCCCCC", -9.853389226361632 ], [ "CCCCCCCCCCCCCCC", -9.888738943788765 ], [ "▁CCCCN", -9.895811636494908 ], [ "▁F", -9.903579345293878 ], [ "▁CCCCCCCCCC", -9.94149907776314 ], [ "ncs", -10.064380400328757 ], [ "NCCC", -10.165294861198284 ], [ "CCCCO", -10.16683340025895 ], [ "CCCCCCCCCCCCCC", -10.239985156271684 ], [ "ns", -10.275063858872905 ], [ "CCCOC", -10.282640047757022 ], [ "▁CCCCCCCCCCO", -10.309135163189584 ], [ "CCSC", -10.363929816625888 ], [ "▁CCCCCCCCCCCC", -10.449414002442618 ], [ "▁CCCO", -10.514011634342324 ], [ "CCOCCO", -10.660929381560551 ], [ "NCCO", -10.703303801457512 ], [ "NN", -10.881162741541972 ], [ "CCCCCCCCCCCCCCCC", -11.100100954370385 ], [ "A", -11.134215822966231 ], [ "p", -11.134215822966231 ], [ "▁CCCCO", -11.220586422056538 ], [ "▁B", -11.28506142857012 ], [ "▁CCOCCC", -11.381007318132896 ], [ "▁CCCCCCCCC", -11.41105455067613 ], [ "OCCC", -11.520210817757922 ], [ "▁CCCCOC", -11.547962564626252 ], [ "CCCCCCC", -11.54852100865634 ], [ "OCCOCCO", -11.77466954306638 ], [ "▁OCC", -12.050092411334065 ], [ "co", -12.244728966155286 ], [ "CCCCCCCC", -12.37163120840584 ], [ "r", -12.414896596223558 ], [ "occ", -12.414896596223558 ] ] } }