add pretrained whisper gpt2 tokenizer

whisper/assets/whisper_mult_gpt2/added_tokens.json

@@ -0,0 +1 @@
+{"<|ta|>": 50287, "<|is|>": 50311, "<|sn|>": 50324, "<|uz|>": 50337, "<|km|>": 50323, "<|lt|>": 50293, "<|bg|>": 50292, "<|mn|>": 50314, "<|lv|>": 50301, "<|th|>": 50289, "<|nl|>": 50271, "<|ja|>": 50266, "<|ha|>": 50354, "<|br|>": 50309, "<|ms|>": 50282, "<|ba|>": 50355, "<|ka|>": 50329, "<|mk|>": 50308, "<|kk|>": 50316, "<|ar|>": 50272, "<|ne|>": 50313, "<|hy|>": 50312, "<|translate|>": 50358, "<|tr|>": 50268, "<|la|>": 50294, "<|gu|>": 50333, "<|sk|>": 50298, "<|my|>": 50346, "<|hr|>": 50291, "<|sa|>": 50344, "<|sq|>": 50317, "<|el|>": 50281, "<|ko|>": 50264, "<|su|>": 50357, "<|oc|>": 50328, "<|kn|>": 50306, "<|te|>": 50299, "<|am|>": 50334, "<|mi|>": 50295, "<|haw|>": 50352, "<|ht|>": 50339, "<|ps|>": 50340, "<|bs|>": 50315, "<|notimestamps|>": 50363, "<|ro|>": 50284, "<|as|>": 50350, "<|eu|>": 50310, "<|si|>": 50322, "<|id|>": 50275, "<|et|>": 50307, "<|startoftranscript|>": 50258, "<|startoflm|>": 50360, "<|no|>": 50288, "<|tt|>": 50351, "<|zh|>": 50260, "<|ca|>": 50270, "<|sl|>": 50305, "<|endoftext|>": 50257, "<|nospeech|>": 50362, "<|cs|>": 50283, "<|fo|>": 50338, "<|cy|>": 50297, "<|lo|>": 50336, "<|tg|>": 50331, "<|fa|>": 50300, "<|ml|>": 50296, "<|es|>": 50262, "<|startofprev|>": 50361, "<|sw|>": 50318, "<|sd|>": 50332, "<|ru|>": 50263, "<|fi|>": 50277, "<|da|>": 50285, "<|en|>": 50259, "<|pt|>": 50267, "<|mr|>": 50320, "<|bo|>": 50347, "<|fr|>": 50265, "<|tl|>": 50348, "<|it|>": 50274, "<|bn|>": 50302, "<|mt|>": 50343, "<|vi|>": 50278, "<|ur|>": 50290, "<|iw|>": 50279, "<|ln|>": 50353, "<|af|>": 50327, "<|transcribe|>": 50359, "<|yo|>": 50325, "<|hi|>": 50276, "<|so|>": 50326, "<|az|>": 50304, "<|jw|>": 50356, "<|sr|>": 50303, "<|tk|>": 50341, "<|uk|>": 50280, "<|hu|>": 50286, "<|gl|>": 50319, "<|pl|>": 50269, "<|de|>": 50261, "<|nn|>": 50342, "<|lb|>": 50345, "<|be|>": 50330, "<|pa|>": 50321, "<|sv|>": 50273, "<|yi|>": 50335, "<|mg|>": 50349}