books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "- added no chat template comparison in test_chat_wrap_and_equivalence"

Browse Source 
This reverts commit 3a5ee8cfa1.
This commit is contained in:
casinca
2025-09-16 09:42:47 +02:00
parent 701b5ad54d
commit 02a1cb1159
1 changed files with 15 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
pkg/llms_from_scratch/tests/test_qwen3.py
View File

@@ -457,19 +457,21 @@ def test_chat_wrap_and_equivalence(add_gen, add_think):
add_thinking=add_think,
)
# Base models: compare raw encoding (no chat template)
if "Base" in repo_id:
ours = qt.encode(prompt) # should use no chat template
ref = hf_tok.encode(prompt) # raw encoding without chat template
else:
# Instruct models: compare with chat template
ours = qt.encode(prompt) # will use chat template
ref = hf_tok.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=add_gen,
enable_thinking=add_think,
)
# Our encode vs HF template
ours = qt.encode(prompt)
ref = hf_tok.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=add_gen,
enable_thinking=add_think,
)
ours = qt.encode(prompt)
ref = hf_tok.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=add_gen,
enable_thinking=add_think,
)
if add_gen and not add_think:
pass # skip edge case as this is not something we use in practice