Upload model trained with Unsloth
Browse filesUpload model trained with Unsloth 2x faster
- added_tokens.json +4 -0
- special_tokens_map.json +14 -2
- tokenizer.json +2 -2
- tokenizer_config.json +19 -3
added_tokens.json
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<fin_univ_x>": 256001,
|
3 |
+
"<inicio_univ_x>": 256000
|
4 |
+
}
|
special_tokens_map.json
CHANGED
@@ -1,7 +1,19 @@
|
|
1 |
{
|
2 |
"additional_special_tokens": [
|
3 |
-
|
4 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
],
|
6 |
"bos_token": {
|
7 |
"content": "<bos>",
|
|
|
1 |
{
|
2 |
"additional_special_tokens": [
|
3 |
+
{
|
4 |
+
"content": "<inicio_univ_x>",
|
5 |
+
"lstrip": false,
|
6 |
+
"normalized": false,
|
7 |
+
"rstrip": false,
|
8 |
+
"single_word": false
|
9 |
+
},
|
10 |
+
{
|
11 |
+
"content": "<fin_univ_x>",
|
12 |
+
"lstrip": false,
|
13 |
+
"normalized": false,
|
14 |
+
"rstrip": false,
|
15 |
+
"single_word": false
|
16 |
+
}
|
17 |
],
|
18 |
"bos_token": {
|
19 |
"content": "<bos>",
|
tokenizer.json
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:bda065ef9089968c02ae493986b8af7e96ba86346164406aaa7c2181c70059f5
|
3 |
+
size 17525738
|
tokenizer_config.json
CHANGED
@@ -1993,18 +1993,34 @@
|
|
1993 |
"rstrip": false,
|
1994 |
"single_word": false,
|
1995 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1996 |
}
|
1997 |
},
|
1998 |
"additional_special_tokens": [
|
1999 |
-
"<
|
2000 |
-
"<
|
2001 |
],
|
2002 |
"bos_token": "<bos>",
|
2003 |
"clean_up_tokenization_spaces": false,
|
2004 |
"eos_token": "<eos>",
|
2005 |
"model_max_length": 8192,
|
2006 |
"pad_token": "<pad>",
|
2007 |
-
"padding_side": "
|
2008 |
"sp_model_kwargs": {},
|
2009 |
"spaces_between_special_tokens": false,
|
2010 |
"tokenizer_class": "GemmaTokenizer",
|
|
|
1993 |
"rstrip": false,
|
1994 |
"single_word": false,
|
1995 |
"special": false
|
1996 |
+
},
|
1997 |
+
"256000": {
|
1998 |
+
"content": "<inicio_univ_x>",
|
1999 |
+
"lstrip": false,
|
2000 |
+
"normalized": false,
|
2001 |
+
"rstrip": false,
|
2002 |
+
"single_word": false,
|
2003 |
+
"special": true
|
2004 |
+
},
|
2005 |
+
"256001": {
|
2006 |
+
"content": "<fin_univ_x>",
|
2007 |
+
"lstrip": false,
|
2008 |
+
"normalized": false,
|
2009 |
+
"rstrip": false,
|
2010 |
+
"single_word": false,
|
2011 |
+
"special": true
|
2012 |
}
|
2013 |
},
|
2014 |
"additional_special_tokens": [
|
2015 |
+
"<inicio_univ_x>",
|
2016 |
+
"<fin_univ_x>"
|
2017 |
],
|
2018 |
"bos_token": "<bos>",
|
2019 |
"clean_up_tokenization_spaces": false,
|
2020 |
"eos_token": "<eos>",
|
2021 |
"model_max_length": 8192,
|
2022 |
"pad_token": "<pad>",
|
2023 |
+
"padding_side": "left",
|
2024 |
"sp_model_kwargs": {},
|
2025 |
"spaces_between_special_tokens": false,
|
2026 |
"tokenizer_class": "GemmaTokenizer",
|