cfm_latn_5mb / added_tokens.json
goldfish-models's picture
Upload cfm_latn_5mb tokenizer.
52466fa
raw
history blame
13.4 kB
{"[XXXXX315]": 36541, "[XXXXX139]": 36365, "[XXXXX81]": 36307, "[XXXXX359]": 36585, "[XXXXX82]": 36308, "[XXXXX164]": 36390, "[XXXXX508]": 36734, "[XXXXX1]": 36227, "[XXXXX220]": 36446, "[XXXXX157]": 36383, "[XXXXX573]": 36799, "[XXXXX347]": 36573, "[XXXXX102]": 36328, "[XXXXX311]": 36537, "[XXXXX313]": 36539, "[XXXXX169]": 36395, "[XXXXX150]": 36376, "[XXXXX55]": 36281, "[XXXXX236]": 36462, "[XXXXX13]": 36239, "[XXXXX412]": 36638, "[XXXXX238]": 36464, "[XXXXX120]": 36346, "[XXXXX295]": 36521, "[XXXXX160]": 36386, "[XXXXX294]": 36520, "[XXXXX624]": 36850, "[XXXXX447]": 36673, "[XXXXX511]": 36737, "[XXXXX86]": 36312, "[XXXXX51]": 36277, "[XXXXX515]": 36741, "[XXXXX443]": 36669, "[XXXXX93]": 36319, "[XXXXX289]": 36515, "[XXXXX402]": 36628, "[XXXXX556]": 36782, "[XXXXX322]": 36548, "[XXXXX430]": 36656, "[XXXXX231]": 36457, "[XXXXX335]": 36561, "[XXXXX436]": 36662, "[XXXXX469]": 36695, "[XXXXX549]": 36775, "[XXXXX482]": 36708, "[XXXXX283]": 36509, "[XXXXX132]": 36358, "[XXXXX321]": 36547, "[XXXXX70]": 36296, "[XXXXX209]": 36435, "[XXXXX79]": 36305, "[XXXXX446]": 36672, "[XXXXX305]": 36531, "[XXXXX352]": 36578, "[XXXXX115]": 36341, "[XXXXX346]": 36572, "[XXXXX571]": 36797, "[XXXXX314]": 36540, "[XXXXX541]": 36767, "[XXXXX588]": 36814, "[XXXXX310]": 36536, "[XXXXX637]": 36863, "[XXXXX218]": 36444, "[XXXXX555]": 36781, "[XXXXX494]": 36720, "[XXXXX131]": 36357, "[XXXXX561]": 36787, "[XXXXX326]": 36552, "[XXXXX336]": 36562, "[XXXXX343]": 36569, "[XXXXX526]": 36752, "[XXXXX544]": 36770, "[XXXXX112]": 36338, "[XXXXX468]": 36694, "[XXXXX374]": 36600, "[XXXXX376]": 36602, "[XXXXX118]": 36344, "[XXXXX423]": 36649, "[XXXXX227]": 36453, "[XXXXX18]": 36244, "[XXXXX466]": 36692, "[XXXXX153]": 36379, "[XXXXX400]": 36626, "[XXXXX431]": 36657, "[XXXXX569]": 36795, "[XXXXX242]": 36468, "[XXXXX123]": 36349, "[XXXXX461]": 36687, "[XXXXX152]": 36378, "[XXXXX291]": 36517, "[XXXXX41]": 36267, "[XXXXX333]": 36559, "[XXXXX594]": 36820, "[XXXXX196]": 36422, "[XXXXX339]": 36565, "[XXXXX149]": 36375, "[XXXXX395]": 36621, "[XXXXX168]": 36394, "[XXXXX539]": 36765, "[XXXXX89]": 36315, "[XXXXX317]": 36543, "[XXXXX274]": 36500, "[XXXXX483]": 36709, "[XXXXX408]": 36634, "[XXXXX582]": 36808, "[XXXXX293]": 36519, "[XXXXX550]": 36776, "[XXXXX226]": 36452, "[XXXXX608]": 36834, "[XXXXX280]": 36506, "[XXXXX547]": 36773, "[XXXXX454]": 36680, "[XXXXX486]": 36712, "[XXXXX319]": 36545, "[XXXXX353]": 36579, "[XXXXX188]": 36414, "[XXXXX216]": 36442, "[XXXXX634]": 36860, "[XXXXX68]": 36294, "[XXXXX344]": 36570, "[XXXXX345]": 36571, "[XXXXX471]": 36697, "[XXXXX567]": 36793, "[XXXXX223]": 36449, "[XXXXX44]": 36270, "[XXXXX351]": 36577, "[XXXXX585]": 36811, "[XXXXX481]": 36707, "[XXXXX434]": 36660, "[XXXXX485]": 36711, "[XXXXX38]": 36264, "[XXXXX453]": 36679, "[XXXXX388]": 36614, "[XXXXX390]": 36616, "[XXXXX182]": 36408, "[XXXXX0]": 36226, "[XXXXX478]": 36704, "[XXXXX382]": 36608, "[XXXXX552]": 36778, "[XXXXX570]": 36796, "[XXXXX512]": 36738, "[XXXXX33]": 36259, "[XXXXX204]": 36430, "[XXXXX20]": 36246, "[XXXXX375]": 36601, "[XXXXX420]": 36646, "[XXXXX26]": 36252, "[XXXXX100]": 36326, "[XXXXX589]": 36815, "[XXXXX435]": 36661, "[MASK]": 36225, "[XXXXX122]": 36348, "[XXXXX308]": 36534, "[XXXXX133]": 36359, "[XXXXX338]": 36564, "[XXXXX213]": 36439, "[XXXXX306]": 36532, "[XXXXX627]": 36853, "[XXXXX427]": 36653, "[XXXXX56]": 36282, "[XXXXX237]": 36463, "[XXXXX380]": 36606, "[XXXXX324]": 36550, "[XXXXX137]": 36363, "[XXXXX97]": 36323, "[XXXXX229]": 36455, "[XXXXX463]": 36689, "[XXXXX29]": 36255, "[XXXXX126]": 36352, "[XXXXX363]": 36589, "[XXXXX614]": 36840, "[XXXXX457]": 36683, "[XXXXX442]": 36668, "[XXXXX531]": 36757, "[XXXXX241]": 36467, "[XXXXX518]": 36744, "[XXXXX71]": 36297, "[XXXXX507]": 36733, "[XXXXX456]": 36682, "[XXXXX385]": 36611, "[XXXXX445]": 36671, "[XXXXX154]": 36380, "[XXXXX381]": 36607, "[XXXXX323]": 36549, "[XXXXX601]": 36827, "[XXXXX63]": 36289, "[XXXXX230]": 36456, "[XXXXX303]": 36529, "[XXXXX356]": 36582, "[XXXXX140]": 36366, "[XXXXX529]": 36755, "[XXXXX203]": 36429, "[XXXXX234]": 36460, "[XXXXX538]": 36764, "[XXXXX177]": 36403, "[XXXXX212]": 36438, "[XXXXX144]": 36370, "[XXXXX564]": 36790, "[XXXXX17]": 36243, "[XXXXX181]": 36407, "[XXXXX426]": 36652, "[XXXXX252]": 36478, "[XXXXX593]": 36819, "[XXXXX117]": 36343, "[XXXXX194]": 36420, "[XXXXX535]": 36761, "[XXXXX40]": 36266, "[XXXXX493]": 36719, "[XXXXX22]": 36248, "[XXXXX247]": 36473, "[XXXXX129]": 36355, "[XXXXX67]": 36293, "[XXXXX586]": 36812, "[XXXXX379]": 36605, "[XXXXX606]": 36832, "[XXXXX190]": 36416, "[XXXXX557]": 36783, "[XXXXX389]": 36615, "[XXXXX158]": 36384, "[XXXXX286]": 36512, "[XXXXX348]": 36574, "[XXXXX183]": 36409, "[XXXXX452]": 36678, "[XXXXX404]": 36630, "[XXXXX58]": 36284, "[XXXXX623]": 36849, "[XXXXX563]": 36789, "[XXXXX179]": 36405, "[XXXXX77]": 36303, "[XXXXX517]": 36743, "[CLS]": 36222, "[XXXXX245]": 36471, "[XXXXX377]": 36603, "[XXXXX127]": 36353, "[XXXXX167]": 36393, "[XXXXX496]": 36722, "[XXXXX334]": 36560, "[XXXXX239]": 36465, "[XXXXX551]": 36777, "[XXXXX612]": 36838, "[XXXXX233]": 36459, "[XXXXX121]": 36347, "[XXXXX208]": 36434, "[XXXXX256]": 36482, "[XXXXX484]": 36710, "[XXXXX258]": 36484, "[XXXXX578]": 36804, "[XXXXX4]": 36230, "[XXXXX170]": 36396, "[XXXXX191]": 36417, "[XXXXX269]": 36495, "[XXXXX340]": 36566, "[XXXXX64]": 36290, "[XXXXX36]": 36262, "[XXXXX525]": 36751, "[XXXXX301]": 36527, "[XXXXX202]": 36428, "[XXXXX479]": 36705, "[XXXXX267]": 36493, "[XXXXX500]": 36726, "[XXXXX49]": 36275, "[XXXXX495]": 36721, "[XXXXX537]": 36763, "[XXXXX146]": 36372, "[XXXXX440]": 36666, "[XXXXX111]": 36337, "[XXXXX625]": 36851, "[XXXXX262]": 36488, "[XXXXX373]": 36599, "[XXXXX610]": 36836, "[XXXXX498]": 36724, "[XXXXX23]": 36249, "[XXXXX110]": 36336, "[XXXXX509]": 36735, "[XXXXX370]": 36596, "[XXXXX592]": 36818, "[XXXXX135]": 36361, "[XXXXX413]": 36639, "[XXXXX628]": 36854, "<pad>": 36224, "[XXXXX475]": 36701, "[XXXXX116]": 36342, "[XXXXX34]": 36260, "[XXXXX159]": 36385, "[XXXXX519]": 36745, "[XXXXX246]": 36472, "[XXXXX394]": 36620, "[XXXXX7]": 36233, "[XXXXX598]": 36824, "[XXXXX211]": 36437, "[XXXXX613]": 36839, "[XXXXX282]": 36508, "[XXXXX198]": 36424, "[XXXXX631]": 36857, "[XXXXX78]": 36304, "[XXXXX148]": 36374, "[XXXXX530]": 36756, "[XXXXX480]": 36706, "[XXXXX35]": 36261, "[XXXXX307]": 36533, "[XXXXX635]": 36861, "[XXXXX114]": 36340, "[XXXXX166]": 36392, "[XXXXX559]": 36785, "[XXXXX225]": 36451, "[XXXXX32]": 36258, "[XXXXX621]": 36847, "[XXXXX69]": 36295, "[XXXXX355]": 36581, "[XXXXX59]": 36285, "[XXXXX476]": 36702, "[XXXXX629]": 36855, "[XXXXX173]": 36399, "[XXXXX337]": 36563, "[XXXXX156]": 36382, "[XXXXX37]": 36263, "[XXXXX165]": 36391, "[XXXXX490]": 36716, "[XXXXX371]": 36597, "[XXXXX278]": 36504, "[XXXXX432]": 36658, "[XXXXX492]": 36718, "[XXXXX417]": 36643, "[XXXXX444]": 36670, "[XXXXX47]": 36273, "[XXXXX19]": 36245, "[XXXXX27]": 36253, "[XXXXX14]": 36240, "[XXXXX72]": 36298, "[XXXXX265]": 36491, "[XXXXX46]": 36272, "[XXXXX96]": 36322, "[XXXXX415]": 36641, "[XXXXX136]": 36362, "[XXXXX309]": 36535, "[XXXXX195]": 36421, "[XXXXX74]": 36300, "[XXXXX533]": 36759, "[XXXXX488]": 36714, "[XXXXX616]": 36842, "[XXXXX90]": 36316, "[XXXXX316]": 36542, "[XXXXX566]": 36792, "[XXXXX617]": 36843, "[XXXXX180]": 36406, "[SEP]": 36223, "[XXXXX411]": 36637, "[XXXXX60]": 36286, "[XXXXX424]": 36650, "[XXXXX192]": 36418, "[XXXXX560]": 36786, "[XXXXX145]": 36371, "[XXXXX106]": 36332, "[XXXXX392]": 36618, "[XXXXX263]": 36489, "[XXXXX503]": 36729, "[XXXXX147]": 36373, "[XXXXX189]": 36415, "[XXXXX397]": 36623, "[XXXXX48]": 36274, "[XXXXX554]": 36780, "[XXXXX162]": 36388, "[XXXXX439]": 36665, "[XXXXX328]": 36554, "[XXXXX43]": 36269, "[XXXXX268]": 36494, "[XXXXX553]": 36779, "[XXXXX255]": 36481, "[XXXXX467]": 36693, "[XXXXX543]": 36769, "[XXXXX540]": 36766, "[XXXXX584]": 36810, "[XXXXX630]": 36856, "[XXXXX171]": 36397, "[XXXXX520]": 36746, "[XXXXX378]": 36604, "[XXXXX607]": 36833, "[XXXXX201]": 36427, "[XXXXX205]": 36431, "[XXXXX590]": 36816, "[XXXXX155]": 36381, "[XXXXX421]": 36647, "[XXXXX425]": 36651, "[XXXXX367]": 36593, "[XXXXX42]": 36268, "[XXXXX618]": 36844, "[XXXXX214]": 36440, "[XXXXX113]": 36339, "[XXXXX207]": 36433, "[XXXXX299]": 36525, "[XXXXX615]": 36841, "[XXXXX522]": 36748, "[XXXXX128]": 36354, "[XXXXX287]": 36513, "[XXXXX405]": 36631, "[XXXXX368]": 36594, "[XXXXX580]": 36806, "[XXXXX357]": 36583, "[XXXXX513]": 36739, "[XXXXX558]": 36784, "[XXXXX450]": 36676, "[XXXXX416]": 36642, "[XXXXX502]": 36728, "[XXXXX365]": 36591, "[XXXXX151]": 36377, "[XXXXX401]": 36627, "[XXXXX174]": 36400, "[XXXXX398]": 36624, "[XXXXX595]": 36821, "[XXXXX565]": 36791, "[XXXXX250]": 36476, "[XXXXX73]": 36299, "[XXXXX364]": 36590, "[XXXXX296]": 36522, "[XXXXX330]": 36556, "[XXXXX163]": 36389, "[XXXXX354]": 36580, "[XXXXX349]": 36575, "[XXXXX85]": 36311, "[XXXXX505]": 36731, "[XXXXX88]": 36314, "[XXXXX599]": 36825, "[XXXXX602]": 36828, "[XXXXX611]": 36837, "[XXXXX240]": 36466, "[XXXXX87]": 36313, "[XXXXX387]": 36613, "[XXXXX587]": 36813, "[XXXXX178]": 36404, "[XXXXX277]": 36503, "[XXXXX632]": 36858, "[XXXXX603]": 36829, "[XXXXX5]": 36231, "[XXXXX596]": 36822, "[XXXXX176]": 36402, "[XXXXX491]": 36717, "[XXXXX399]": 36625, "[XXXXX219]": 36445, "[XXXXX31]": 36257, "[XXXXX419]": 36645, "[XXXXX633]": 36859, "[XXXXX341]": 36567, "[XXXXX21]": 36247, "[XXXXX572]": 36798, "[XXXXX523]": 36749, "[XXXXX138]": 36364, "[XXXXX25]": 36251, "[XXXXX465]": 36691, "[XXXXX433]": 36659, "[XXXXX6]": 36232, "[XXXXX597]": 36823, "[XXXXX329]": 36555, "[XXXXX428]": 36654, "[XXXXX414]": 36640, "[XXXXX304]": 36530, "[XXXXX576]": 36802, "[XXXXX30]": 36256, "[XXXXX300]": 36526, "[XXXXX175]": 36401, "[XXXXX619]": 36845, "[XXXXX24]": 36250, "[XXXXX568]": 36794, "[XXXXX532]": 36758, "[XXXXX472]": 36698, "[XXXXX504]": 36730, "[XXXXX206]": 36432, "[XXXXX200]": 36426, "[XXXXX406]": 36632, "[XXXXX109]": 36335, "[XXXXX327]": 36553, "[XXXXX143]": 36369, "[XXXXX271]": 36497, "[XXXXX184]": 36410, "[XXXXX125]": 36351, "[XXXXX487]": 36713, "[XXXXX284]": 36510, "[XXXXX470]": 36696, "[XXXXX9]": 36235, "[XXXXX76]": 36302, "[XXXXX403]": 36629, "[XXXXX383]": 36609, "[XXXXX3]": 36229, "[XXXXX187]": 36413, "[XXXXX386]": 36612, "[XXXXX107]": 36333, "[XXXXX254]": 36480, "[XXXXX464]": 36690, "[XXXXX429]": 36655, "[XXXXX52]": 36278, "[XXXXX534]": 36760, "[XXXXX228]": 36454, "[XXXXX546]": 36772, "[XXXXX501]": 36727, "[XXXXX62]": 36288, "[XXXXX622]": 36848, "[XXXXX243]": 36469, "[XXXXX11]": 36237, "[XXXXX276]": 36502, "[XXXXX462]": 36688, "[XXXXX369]": 36595, "[XXXXX536]": 36762, "[XXXXX39]": 36265, "[XXXXX142]": 36368, "[XXXXX8]": 36234, "[XXXXX83]": 36309, "[XXXXX455]": 36681, "[XXXXX251]": 36477, "[XXXXX473]": 36699, "[XXXXX358]": 36584, "[XXXXX527]": 36753, "[XXXXX342]": 36568, "[XXXXX477]": 36703, "[XXXXX185]": 36411, "[XXXXX331]": 36557, "[XXXXX217]": 36443, "[XXXXX499]": 36725, "[XXXXX528]": 36754, "[XXXXX257]": 36483, "[XXXXX12]": 36238, "[XXXXX636]": 36862, "[XXXXX253]": 36479, "[XXXXX581]": 36807, "[XXXXX441]": 36667, "[XXXXX266]": 36492, "[XXXXX75]": 36301, "[XXXXX105]": 36331, "[XXXXX222]": 36448, "[XXXXX562]": 36788, "[XXXXX489]": 36715, "[XXXXX259]": 36485, "[XXXXX215]": 36441, "[XXXXX290]": 36516, "[XXXXX605]": 36831, "[XXXXX298]": 36524, "[XXXXX604]": 36830, "[XXXXX288]": 36514, "[XXXXX409]": 36635, "[XXXXX235]": 36461, "[XXXXX510]": 36736, "[XXXXX575]": 36801, "[XXXXX297]": 36523, "[XXXXX224]": 36450, "[XXXXX172]": 36398, "[XXXXX92]": 36318, "[XXXXX438]": 36664, "[XXXXX28]": 36254, "[XXXXX130]": 36356, "[XXXXX91]": 36317, "[XXXXX50]": 36276, "[XXXXX396]": 36622, "[XXXXX199]": 36425, "[XXXXX514]": 36740, "[XXXXX260]": 36486, "[XXXXX609]": 36835, "[XXXXX66]": 36292, "[XXXXX372]": 36598, "[XXXXX366]": 36592, "[XXXXX583]": 36809, "[XXXXX542]": 36768, "[XXXXX65]": 36291, "[XXXXX15]": 36241, "[XXXXX248]": 36474, "[XXXXX410]": 36636, "[XXXXX524]": 36750, "[XXXXX460]": 36686, "[XXXXX320]": 36546, "[XXXXX574]": 36800, "[XXXXX516]": 36742, "[XXXXX193]": 36419, "[XXXXX273]": 36499, "[XXXXX57]": 36283, "[XXXXX292]": 36518, "[XXXXX261]": 36487, "[XXXXX249]": 36475, "[XXXXX451]": 36677, "[XXXXX221]": 36447, "[XXXXX577]": 36803, "[XXXXX506]": 36732, "[XXXXX548]": 36774, "[XXXXX94]": 36320, "[XXXXX80]": 36306, "[XXXXX232]": 36458, "[XXXXX521]": 36747, "[XXXXX422]": 36648, "[XXXXX497]": 36723, "[XXXXX449]": 36675, "[XXXXX84]": 36310, "[XXXXX279]": 36505, "[XXXXX362]": 36588, "[XXXXX620]": 36846, "[XXXXX318]": 36544, "[XXXXX16]": 36242, "[XXXXX270]": 36496, "[XXXXX285]": 36511, "[XXXXX579]": 36805, "[XXXXX45]": 36271, "[XXXXX119]": 36345, "[XXXXX244]": 36470, "[XXXXX545]": 36771, "[XXXXX197]": 36423, "[XXXXX391]": 36617, "[XXXXX141]": 36367, "[XXXXX99]": 36325, "[XXXXX312]": 36538, "[XXXXX332]": 36558, "[XXXXX361]": 36587, "[XXXXX98]": 36324, "[XXXXX161]": 36387, "[XXXXX384]": 36610, "[XXXXX360]": 36586, "[XXXXX264]": 36490, "[XXXXX393]": 36619, "[XXXXX302]": 36528, "[XXXXX474]": 36700, "[XXXXX437]": 36663, "[XXXXX95]": 36321, "[XXXXX418]": 36644, "[XXXXX101]": 36327, "[XXXXX61]": 36287, "[XXXXX134]": 36360, "[XXXXX10]": 36236, "[XXXXX350]": 36576, "[XXXXX459]": 36685, "[XXXXX103]": 36329, "[XXXXX275]": 36501, "[XXXXX458]": 36684, "[XXXXX104]": 36330, "[XXXXX2]": 36228, "[XXXXX626]": 36852, "[XXXXX53]": 36279, "[XXXXX124]": 36350, "[XXXXX407]": 36633, "[XXXXX210]": 36436, "[XXXXX108]": 36334, "[XXXXX325]": 36551, "[XXXXX600]": 36826, "[XXXXX591]": 36817, "[XXXXX448]": 36674, "[XXXXX272]": 36498, "[XXXXX281]": 36507, "[XXXXX186]": 36412, "[XXXXX54]": 36280}