books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed num_workers (#229)

Browse Source 
* fixed num_workers

* ch06 & ch07: added num_workers to create_dataloader_v1
This commit is contained in:
Daniel Kleine
2024-06-20 00:36:46 +02:00
committed by GitHub
parent 24523bd34d
commit bbb2a0c3d5
15 changed files with 20 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ch06/02_bonus_additional-experiments/previous_chapters.py
View File

@@ -41,7 +41,7 @@ class GPTDatasetV1(Dataset):
def create_dataloader_v1(txt, batch_size=4, max_length=256,
stride=128, shuffle=True, drop_last=True):
stride=128, shuffle=True, drop_last=True, num_workers=0):
# Initialize the tokenizer
tokenizer = tiktoken.get_encoding("gpt2")
@@ -50,7 +50,7 @@ def create_dataloader_v1(txt, batch_size=4, max_length=256,
# Create dataloader
dataloader = DataLoader(
dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)
dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=num_workers)
return dataloader