books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

prevent self.apply_chat_template being applied for base Qwen models

Browse Source
This commit is contained in:
casinca
2025-09-15 16:26:17 +02:00
parent 8add26cbe9
commit df504397a8
5 changed files with 27 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/llms_from_scratch/qwen3.py
View File

@@ -531,6 +531,7 @@ class Qwen3Tokenizer:
self.apply_chat_template = apply_chat_template
self.add_generation_prompt = add_generation_prompt
self.add_thinking = add_thinking
self.repo_id=repo_id
tok_file = Path(tokenizer_file_path)
if not tok_file.is_file() and repo_id:
@@ -549,7 +550,7 @@ class Qwen3Tokenizer:
self.pad_token_id = self._special_to_id["<|endoftext|>"]
self.eos_token_id = self.pad_token_id
if repo_id and "Base" not in repo_id:
if self.repo_id and "Base" not in self.repo_id:
eos_token = "<|im_end|>"
else:
eos_token = "<|endoftext|>"
@@ -557,7 +558,7 @@ class Qwen3Tokenizer:
self.eos_token_id = self._special_to_id[eos_token]
def encode(self, text, chat_wrapped=None):
if chat_wrapped is None:
if chat_wrapped is None and self.repo_id and "Base" not in self.repo_id:
chat_wrapped = self.apply_chat_template
stripped = text.strip()