books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Qwen3 tokenizer sanity checks (#730)

Browse Source
This commit is contained in:
Sebastian Raschka
2025-07-09 13:52:35 -05:00
committed by GitHub
parent d23b1f07b8
commit 7dc1dcbe27
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/llms_from_scratch/tests/test_qwen3.py
View File

@@ -279,3 +279,11 @@ def test_tokenizer_equivalence():
assert tokenizer_ref.eos_token_id == tokenizer.eos_token_id
assert tokenizer_ref.pad_token_id == tokenizer.pad_token_id
assert tokenizer.encode("<|endoftext|>") == [tokenizer._special_to_id["<|endoftext|>"]]
assert tokenizer.encode("<|im_end|>") == [tokenizer._special_to_id["<|im_end|>"]]
expected_eos_token = "<|im_end|>" if "Base" not in repo_id else "<|endoftext|>"
expected_pad_token = "<|endoftext|>"
assert tokenizer.decode([tokenizer.eos_token_id]) == expected_eos_token
assert tokenizer.decode([tokenizer.pad_token_id]) == expected_pad_token