mirror of
https://github.com/aladdinpersson/Machine-Learning-Collection.git
synced 2026-02-21 11:18:01 +00:00
8 lines
271 B
Python
8 lines
271 B
Python
from transformers import WhisperTokenizer
|
|
tokenizer = WhisperTokenizer.from_pretrained(
|
|
f"openai/whisper-tiny", task="transcribe"
|
|
)
|
|
encoded_string = tokenizer.encode("")[0]
|
|
print(encoded_string) # should print 50258
|
|
print(tokenizer.bos_token_id) # should print 50257
|