books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add backup URL for gpt2 weights (#469)

Browse Source 
* Add backup URL for gpt2 weights

* newline
This commit is contained in:
Sebastian Raschka
2025-01-05 11:28:09 -06:00
committed by GitHub
parent 9b95557ba2
commit 701090815e
8 changed files with 395 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
ch05/01_main-chapter-code/gpt_download.py
View File

@@ -23,6 +23,7 @@ def download_and_load_gpt2(model_size, models_dir):
# Define paths
model_dir = os.path.join(models_dir, model_size)
base_url = "https://openaipublic.blob.core.windows.net/gpt-2/models"
backup_base_url = "https://f001.backblazeb2.com/file/LLMs-from-scratch/gpt2"
filenames = [
"checkpoint", "encoder.json", "hparams.json",
"model.ckpt.data-00000-of-00001", "model.ckpt.index",
@@ -33,8 +34,9 @@ def download_and_load_gpt2(model_size, models_dir):
os.makedirs(model_dir, exist_ok=True)
for filename in filenames:
file_url = os.path.join(base_url, model_size, filename)
backup_url = os.path.join(backup_base_url, model_size, filename)
file_path = os.path.join(model_dir, filename)
download_file(file_url, file_path)
download_file(file_url, file_path, backup_url)
# Load settings and params
tf_ckpt_path = tf.train.latest_checkpoint(model_dir)
@@ -44,11 +46,9 @@ def download_and_load_gpt2(model_size, models_dir):
return settings, params
def download_file(url, destination):
# Send a GET request to download the file
try:
with urllib.request.urlopen(url) as response:
def download_file(url, destination, backup_url=None):
def _attempt_download(download_url):
with urllib.request.urlopen(download_url) as response:
# Get the total file size from headers, defaulting to 0 if not present
file_size = int(response.headers.get("Content-Length", 0))
@@ -57,29 +57,44 @@ def download_file(url, destination):
file_size_local = os.path.getsize(destination)
if file_size == file_size_local:
print(f"File already exists and is up-to-date: {destination}")
return
return True # Indicate success without re-downloading
# Define the block size for reading the file
block_size = 1024 # 1 Kilobyte
# Initialize the progress bar with total file size
progress_bar_description = os.path.basename(url) # Extract filename from URL
progress_bar_description = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=progress_bar_description) as progress_bar:
# Open the destination file in binary write mode
with open(destination, "wb") as file:
# Read the file in chunks and write to destination
while True:
chunk = response.read(block_size)
if not chunk:
break
file.write(chunk)
progress_bar.update(len(chunk)) # Update progress bar
except urllib.error.HTTPError:
s = (
f"The specified URL ({url}) is incorrect, the internet connection cannot be established,"
"\nor the requested file is temporarily unavailable.\nPlease visit the following website"
" for help: https://github.com/rasbt/LLMs-from-scratch/discussions/273")
print(s)
progress_bar.update(len(chunk))
return True
try:
if _attempt_download(url):
return
except (urllib.error.HTTPError, urllib.error.URLError):
if backup_url is not None:
print(f"Primary URL ({url}) failed. Attempting backup URL: {backup_url}")
try:
if _attempt_download(backup_url):
return
except urllib.error.HTTPError:
pass
# If we reach here, both attempts have failed
error_message = (
f"Failed to download from both primary URL ({url})"
f"{' and backup URL (' + backup_url + ')' if backup_url else ''}."
"\nCheck your internet connection or the file availability.\n"
"For help, visit: https://github.com/rasbt/LLMs-from-scratch/discussions/273"
)
print(error_message)
except Exception as e:
print(f"An unexpected error occurred: {e}")
# Alternative way using `requests`

61
ch05/01_main-chapter-code/tests.py
View File

@@ -63,36 +63,39 @@ def check_file_size(url, expected_size):
def test_model_files():
base_url = "https://openaipublic.blob.core.windows.net/gpt-2/models"
def check_model_files(base_url):
model_size = "124M"
files = {
"checkpoint": 77,
"encoder.json": 1042301,
"hparams.json": 90,
"model.ckpt.data-00000-of-00001": 497759232,
"model.ckpt.index": 5215,
"model.ckpt.meta": 471155,
"vocab.bpe": 456318
}
model_size = "124M"
files = {
"checkpoint": 77,
"encoder.json": 1042301,
"hparams.json": 90,
"model.ckpt.data-00000-of-00001": 497759232,
"model.ckpt.index": 5215,
"model.ckpt.meta": 471155,
"vocab.bpe": 456318
}
for file_name, expected_size in files.items():
url = f"{base_url}/{model_size}/{file_name}"
valid, message = check_file_size(url, expected_size)
assert valid, message
for file_name, expected_size in files.items():
url = f"{base_url}/{model_size}/{file_name}"
valid, message = check_file_size(url, expected_size)
assert valid, message
model_size = "355M"
files = {
"checkpoint": 77,
"encoder.json": 1042301,
"hparams.json": 91,
"model.ckpt.data-00000-of-00001": 1419292672,
"model.ckpt.index": 10399,
"model.ckpt.meta": 926519,
"vocab.bpe": 456318
}
model_size = "355M"
files = {
"checkpoint": 77,
"encoder.json": 1042301,
"hparams.json": 91,
"model.ckpt.data-00000-of-00001": 1419292672,
"model.ckpt.index": 10399,
"model.ckpt.meta": 926519,
"vocab.bpe": 456318
}
for file_name, expected_size in files.items():
url = f"{base_url}/{model_size}/{file_name}"
valid, message = check_file_size(url, expected_size)
assert valid, message
for file_name, expected_size in files.items():
url = f"{base_url}/{model_size}/{file_name}"
valid, message = check_file_size(url, expected_size)
assert valid, message
check_model_files(base_url="https://openaipublic.blob.core.windows.net/gpt-2/models")
check_model_files(base_url="https://f001.backblazeb2.com/file/LLMs-from-scratch/gpt2")