mirror of
https://github.com/rasbt/LLMs-from-scratch.git
synced 2026-04-10 12:33:42 +00:00
Requirements update (#851)
* Requirements update * Code change to tricker workers * update
This commit is contained in:
committed by
GitHub
parent
322000d833
commit
f492c949d3
@@ -33,6 +33,7 @@ class GPTDatasetV1(Dataset):
|
||||
|
||||
def create_dataloader_v1(txt, batch_size=4, max_length=256,
|
||||
stride=128, shuffle=True, drop_last=True, num_workers=0):
|
||||
|
||||
# Initialize the tokenizer
|
||||
tokenizer = tiktoken.get_encoding("gpt2")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user