add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"[ANSW]": 50264, "[eot]": 50265, "[PAD]": 50257, "[TEXT]": 50263, "<|answer|>": 50260, "[QUES]": 50262, "<|text|>": 50259, "<|question|>": 50258, "<|answer_sep|>": 50261}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>", "pad_token": "[PAD]", "additional_special_tokens": ["<|question|>", "<|text|>", "<|answer|>", "<|answer_sep|>", "[QUES]", "[TEXT]", "[ANSW]", "[eot]"]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "use_fast": true, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "/Users/stefanhg/Documents/UMich/Research/Zeroshot Text Classification/models-upload/2022-06-19_13-08-17_NVIDIA-GPT2-gpt2-medium-vanilla-aspect-norm/trained", "tokenizer_class": "ZsGPT2Tokenizer"}