books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix issue 724: unused args (#726)

Browse Source 
* Fix issue 724: unused args

* Update 02_opt_multi_gpu_ddp.py
This commit is contained in:
Matthew Hernandez
2025-07-08 04:37:39 -07:00
committed by rasbt
parent 5188a41267
commit 1552023bd4
1 changed files with 0 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
ch02/05_bpe-from-scratch/tests.py
View File

@@ -56,22 +56,6 @@ def verdict_file(imported_module):
return paths
@pytest.fixture(scope="module")
def gpt2_files(imported_module):
"""Fixture to handle downloading GPT-2 files."""
download_file_if_absent = getattr(imported_module, "download_file_if_absent", None)
search_directories = ["ch02/02_bonus_bytepair-encoder/gpt2_model/", "../02_bonus_bytepair-encoder/gpt2_model/", "."]
files_to_download = {
"https://openaipublic.blob.core.windows.net/gpt-2/models/124M/vocab.bpe": "vocab.bpe",
"https://openaipublic.blob.core.windows.net/gpt-2/models/124M/encoder.json": "encoder.json",
}
paths = {filename: download_file_if_absent(url, filename, search_directories)
for url, filename in files_to_download.items()}
return paths
def test_tokenizer_training(imported_module, verdict_file):
BPETokenizerSimple = getattr(imported_module, "BPETokenizerSimple", None)
tokenizer = BPETokenizerSimple()