books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "prevent self.apply_chat_template being applied for base Qwen models"

Browse Source 
This reverts commit df504397a8.
This commit is contained in:
casinca
2025-09-16 09:43:01 +02:00
parent 02a1cb1159
commit 186e83c579
5 changed files with 22 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ch05/11_qwen3/standalone-qwen3-moe-plus-kvcache.ipynb
View File

@@ -973,7 +973,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 18,
"id": "b68ab489-48e5-471e-a814-56cda2d60f81",
"metadata": {},
"outputs": [],
@@ -1000,7 +1000,6 @@
" self.apply_chat_template = apply_chat_template\n",
" self.add_generation_prompt = add_generation_prompt\n",
" self.add_thinking = add_thinking\n",
" self.repo_id=repo_id\n",
"\n",
" tok_file = Path(tokenizer_file_path)\n",
" self._tok = Tokenizer.from_file(str(tok_file))\n",
@@ -1013,7 +1012,7 @@
" self.pad_token_id = self._special_to_id[\"<|endoftext|>\"]\n",
" self.eos_token_id = self.pad_token_id\n",
"\n",
" if self.repo_id and \"Base\" not in self.repo_id:\n",
" if repo_id and \"Base\" not in repo_id:\n",
" eos_token = \"<|im_end|>\"\n",
" else:\n",
" eos_token = \"<|endoftext|>\"\n",
@@ -1021,7 +1020,7 @@
" self.eos_token_id = self._special_to_id[eos_token]\n",
"\n",
" def encode(self, text, chat_wrapped=None):\n",
" if chat_wrapped is None and self.repo_id and \"Base\" not in self.repo_id:\n",
" if chat_wrapped is None:\n",
" chat_wrapped = self.apply_chat_template\n",
"\n",
" stripped = text.strip()\n",
@@ -1240,7 +1239,7 @@
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -1254,7 +1253,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.6"
"version": "3.10.16"
}
},
"nbformat": 4,

11
ch05/11_qwen3/standalone-qwen3-moe.ipynb
View File

@@ -915,7 +915,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 17,
"id": "b68ab489-48e5-471e-a814-56cda2d60f81",
"metadata": {},
"outputs": [],
@@ -942,7 +942,6 @@
" self.apply_chat_template = apply_chat_template\n",
" self.add_generation_prompt = add_generation_prompt\n",
" self.add_thinking = add_thinking\n",
" self.repo_id=repo_id\n",
"\n",
" tok_file = Path(tokenizer_file_path)\n",
" self._tok = Tokenizer.from_file(str(tok_file))\n",
@@ -955,7 +954,7 @@
" self.pad_token_id = self._special_to_id[\"<|endoftext|>\"]\n",
" self.eos_token_id = self.pad_token_id\n",
"\n",
" if self.repo_id and \"Base\" not in self.repo_id:\n",
" if repo_id and \"Base\" not in repo_id:\n",
" eos_token = \"<|im_end|>\"\n",
" else:\n",
" eos_token = \"<|endoftext|>\"\n",
@@ -963,7 +962,7 @@
" self.eos_token_id = self._special_to_id[eos_token]\n",
"\n",
" def encode(self, text, chat_wrapped=None):\n",
" if chat_wrapped is None and self.repo_id and \"Base\" not in self.repo_id:\n",
" if chat_wrapped is None:\n",
" chat_wrapped = self.apply_chat_template\n",
"\n",
" stripped = text.strip()\n",
@@ -1222,7 +1221,7 @@
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -1236,7 +1235,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.6"
"version": "3.10.16"
}
},
"nbformat": 4,

11
ch05/11_qwen3/standalone-qwen3-plus-kvcache.ipynb
View File

@@ -960,7 +960,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 20,
"id": "b68ab489-48e5-471e-a814-56cda2d60f81",
"metadata": {},
"outputs": [],
@@ -987,7 +987,6 @@
" self.apply_chat_template = apply_chat_template\n",
" self.add_generation_prompt = add_generation_prompt\n",
" self.add_thinking = add_thinking\n",
" self.repo_id=repo_id\n",
"\n",
" tok_file = Path(tokenizer_file_path)\n",
" self._tok = Tokenizer.from_file(str(tok_file))\n",
@@ -1000,7 +999,7 @@
" self.pad_token_id = self._special_to_id[\"<|endoftext|>\"]\n",
" self.eos_token_id = self.pad_token_id\n",
"\n",
" if self.repo_id and \"Base\" not in self.repo_id:\n",
" if repo_id and \"Base\" not in repo_id:\n",
" eos_token = \"<|im_end|>\"\n",
" else:\n",
" eos_token = \"<|endoftext|>\"\n",
@@ -1008,7 +1007,7 @@
" self.eos_token_id = self._special_to_id[eos_token]\n",
"\n",
" def encode(self, text, chat_wrapped=None):\n",
" if chat_wrapped is None and self.repo_id and \"Base\" not in self.repo_id:\n",
" if chat_wrapped is None:\n",
" chat_wrapped = self.apply_chat_template\n",
"\n",
" stripped = text.strip()\n",
@@ -1207,7 +1206,7 @@
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -1221,7 +1220,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.6"
"version": "3.10.16"
}
},
"nbformat": 4,

11
ch05/11_qwen3/standalone-qwen3.ipynb
View File

@@ -902,7 +902,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 19,
"id": "b68ab489-48e5-471e-a814-56cda2d60f81",
"metadata": {},
"outputs": [],
@@ -929,7 +929,6 @@
" self.apply_chat_template = apply_chat_template\n",
" self.add_generation_prompt = add_generation_prompt\n",
" self.add_thinking = add_thinking\n",
" self.repo_id=repo_id\n",
"\n",
" tok_file = Path(tokenizer_file_path)\n",
" self._tok = Tokenizer.from_file(str(tok_file))\n",
@@ -942,7 +941,7 @@
" self.pad_token_id = self._special_to_id[\"<|endoftext|>\"]\n",
" self.eos_token_id = self.pad_token_id\n",
"\n",
" if self.repo_id and \"Base\" not in self.repo_id:\n",
" if repo_id and \"Base\" not in repo_id:\n",
" eos_token = \"<|im_end|>\"\n",
" else:\n",
" eos_token = \"<|endoftext|>\"\n",
@@ -950,7 +949,7 @@
" self.eos_token_id = self._special_to_id[eos_token]\n",
"\n",
" def encode(self, text, chat_wrapped=None):\n",
" if chat_wrapped is None and self.repo_id and \"Base\" not in self.repo_id:\n",
" if chat_wrapped is None:\n",
" chat_wrapped = self.apply_chat_template\n",
"\n",
" stripped = text.strip()\n",
@@ -1142,7 +1141,7 @@
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -1156,7 +1155,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.6"
"version": "3.10.16"
}
},
"nbformat": 4,

5
pkg/llms_from_scratch/qwen3.py
View File

@@ -531,7 +531,6 @@ class Qwen3Tokenizer:
self.apply_chat_template = apply_chat_template
self.add_generation_prompt = add_generation_prompt
self.add_thinking = add_thinking
self.repo_id=repo_id
tok_file = Path(tokenizer_file_path)
if not tok_file.is_file() and repo_id:
@@ -550,7 +549,7 @@ class Qwen3Tokenizer:
self.pad_token_id = self._special_to_id["<|endoftext|>"]
self.eos_token_id = self.pad_token_id
if self.repo_id and "Base" not in self.repo_id:
if repo_id and "Base" not in repo_id:
eos_token = "<|im_end|>"
else:
eos_token = "<|endoftext|>"
@@ -558,7 +557,7 @@ class Qwen3Tokenizer:
self.eos_token_id = self._special_to_id[eos_token]
def encode(self, text, chat_wrapped=None):
if chat_wrapped is None and self.repo_id and "Base" not in self.repo_id:
if chat_wrapped is None:
chat_wrapped = self.apply_chat_template
stripped = text.strip()