books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "prevent self.apply_chat_template being applied for base Qwen models"

Browse Source 
This reverts commit df504397a8.
This commit is contained in:
casinca
2025-09-16 09:43:01 +02:00
parent 02a1cb1159
commit 186e83c579
5 changed files with 22 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/llms_from_scratch/qwen3.py
View File

@@ -531,7 +531,6 @@ class Qwen3Tokenizer:
self.apply_chat_template = apply_chat_template
self.add_generation_prompt = add_generation_prompt
self.add_thinking = add_thinking
self.repo_id=repo_id
tok_file = Path(tokenizer_file_path)
if not tok_file.is_file() and repo_id:
@@ -550,7 +549,7 @@ class Qwen3Tokenizer:
self.pad_token_id = self._special_to_id["<|endoftext|>"]
self.eos_token_id = self.pad_token_id
if self.repo_id and "Base" not in self.repo_id:
if repo_id and "Base" not in repo_id:
eos_token = "<|im_end|>"
else:
eos_token = "<|endoftext|>"
@@ -558,7 +557,7 @@ class Qwen3Tokenizer:
self.eos_token_id = self._special_to_id[eos_token]
def encode(self, text, chat_wrapped=None):
if chat_wrapped is None and self.repo_id and "Base" not in self.repo_id:
if chat_wrapped is None:
chat_wrapped = self.apply_chat_template
stripped = text.strip()