mirror of
https://github.com/rasbt/LLMs-from-scratch.git
synced 2026-04-10 12:33:42 +00:00
CPU compile performance for Qwen3 models (#704)
* Ch06 classifier function asserts * Qwen3 cpu compilation perf
This commit is contained in:
committed by
GitHub
parent
4014bdd520
commit
37b26c2e04
@@ -309,7 +309,7 @@ class Llama3Tokenizer:
|
||||
special_tokens=self.special,
|
||||
)
|
||||
|
||||
def encode(self, text, bos=False, eos=False):
|
||||
def encode(self, text, bos=False, eos=False, **kwargs):
|
||||
ids = ([self.special["<|begin_of_text|>"]] if bos else []) \
|
||||
+ self.model.encode(text)
|
||||
if eos:
|
||||
|
||||
Reference in New Issue
Block a user