books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Swap urllib.request with requests (#993)

Browse Source
This commit is contained in:
Sebastian Raschka
2026-03-30 22:03:13 -04:00
committed by GitHub
parent 6b9502056f
commit d977841fad
4 changed files with 76 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
appendix-E/01_main-chapter-code/gpt_download.py
View File

@@ -5,11 +5,9 @@
import os
import urllib.request
# import requests
import json
import numpy as np
import requests
import tensorflow as tf
from tqdm import tqdm
@@ -48,27 +46,24 @@ def download_and_load_gpt2(model_size, models_dir):
def download_file(url, destination, backup_url=None):
def _attempt_download(download_url):
with urllib.request.urlopen(download_url) as response:
# Get the total file size from headers, defaulting to 0 if not present
response = requests.get(download_url, stream=True, timeout=60)
response.raise_for_status()
file_size = int(response.headers.get("Content-Length", 0))
# Check if file exists and has the same size
# Check if file exists and has same size
if os.path.exists(destination):
file_size_local = os.path.getsize(destination)
if file_size == file_size_local:
if file_size and file_size == file_size_local:
print(f"File already exists and is up-to-date: {destination}")
return True # Indicate success without re-downloading
return True
block_size = 1024 # 1 Kilobyte
# Initialize the progress bar with total file size
progress_bar_description = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=progress_bar_description) as progress_bar:
block_size = 1024 # 1 KB
desc = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=desc) as progress_bar:
with open(destination, "wb") as file:
while True:
chunk = response.read(block_size)
if not chunk:
break
for chunk in response.iter_content(chunk_size=block_size):
if chunk:
file.write(chunk)
progress_bar.update(len(chunk))
return True
@@ -76,13 +71,13 @@ def download_file(url, destination, backup_url=None):
try:
if _attempt_download(url):
return
except (urllib.error.HTTPError, urllib.error.URLError):
except requests.exceptions.RequestException:
if backup_url is not None:
print(f"Primary URL ({url}) failed. Attempting backup URL: {backup_url}")
try:
if _attempt_download(backup_url):
return
except urllib.error.HTTPError:
except requests.exceptions.RequestException:
pass
# If we reach here, both attempts have failed

33
ch06/01_main-chapter-code/gpt_download.py
View File

@@ -5,11 +5,9 @@
import os
import urllib.request
# import requests
import json
import numpy as np
import requests
import tensorflow as tf
from tqdm import tqdm
@@ -48,27 +46,24 @@ def download_and_load_gpt2(model_size, models_dir):
def download_file(url, destination, backup_url=None):
def _attempt_download(download_url):
with urllib.request.urlopen(download_url) as response:
# Get the total file size from headers, defaulting to 0 if not present
response = requests.get(download_url, stream=True, timeout=60)
response.raise_for_status()
file_size = int(response.headers.get("Content-Length", 0))
# Check if file exists and has the same size
# Check if file exists and has same size
if os.path.exists(destination):
file_size_local = os.path.getsize(destination)
if file_size == file_size_local:
if file_size and file_size == file_size_local:
print(f"File already exists and is up-to-date: {destination}")
return True # Indicate success without re-downloading
return True
block_size = 1024 # 1 Kilobyte
# Initialize the progress bar with total file size
progress_bar_description = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=progress_bar_description) as progress_bar:
block_size = 1024 # 1 KB
desc = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=desc) as progress_bar:
with open(destination, "wb") as file:
while True:
chunk = response.read(block_size)
if not chunk:
break
for chunk in response.iter_content(chunk_size=block_size):
if chunk:
file.write(chunk)
progress_bar.update(len(chunk))
return True
@@ -76,13 +71,13 @@ def download_file(url, destination, backup_url=None):
try:
if _attempt_download(url):
return
except (urllib.error.HTTPError, urllib.error.URLError):
except requests.exceptions.RequestException:
if backup_url is not None:
print(f"Primary URL ({url}) failed. Attempting backup URL: {backup_url}")
try:
if _attempt_download(backup_url):
return
except urllib.error.HTTPError:
except requests.exceptions.RequestException:
pass
# If we reach here, both attempts have failed

33
ch06/02_bonus_additional-experiments/gpt_download.py
View File

@@ -5,11 +5,9 @@
import os
import urllib.request
# import requests
import json
import numpy as np
import requests
import tensorflow as tf
from tqdm import tqdm
@@ -48,27 +46,24 @@ def download_and_load_gpt2(model_size, models_dir):
def download_file(url, destination, backup_url=None):
def _attempt_download(download_url):
with urllib.request.urlopen(download_url) as response:
# Get the total file size from headers, defaulting to 0 if not present
response = requests.get(download_url, stream=True, timeout=60)
response.raise_for_status()
file_size = int(response.headers.get("Content-Length", 0))
# Check if file exists and has the same size
# Check if file exists and has same size
if os.path.exists(destination):
file_size_local = os.path.getsize(destination)
if file_size == file_size_local:
if file_size and file_size == file_size_local:
print(f"File already exists and is up-to-date: {destination}")
return True # Indicate success without re-downloading
return True
block_size = 1024 # 1 Kilobyte
# Initialize the progress bar with total file size
progress_bar_description = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=progress_bar_description) as progress_bar:
block_size = 1024 # 1 KB
desc = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=desc) as progress_bar:
with open(destination, "wb") as file:
while True:
chunk = response.read(block_size)
if not chunk:
break
for chunk in response.iter_content(chunk_size=block_size):
if chunk:
file.write(chunk)
progress_bar.update(len(chunk))
return True
@@ -76,13 +71,13 @@ def download_file(url, destination, backup_url=None):
try:
if _attempt_download(url):
return
except (urllib.error.HTTPError, urllib.error.URLError):
except requests.exceptions.RequestException:
if backup_url is not None:
print(f"Primary URL ({url}) failed. Attempting backup URL: {backup_url}")
try:
if _attempt_download(backup_url):
return
except urllib.error.HTTPError:
except requests.exceptions.RequestException:
pass
# If we reach here, both attempts have failed

33
ch06/03_bonus_imdb-classification/gpt_download.py
View File

@@ -5,11 +5,9 @@
import os
import urllib.request
# import requests
import json
import numpy as np
import requests
import tensorflow as tf
from tqdm import tqdm
@@ -48,27 +46,24 @@ def download_and_load_gpt2(model_size, models_dir):
def download_file(url, destination, backup_url=None):
def _attempt_download(download_url):
with urllib.request.urlopen(download_url) as response:
# Get the total file size from headers, defaulting to 0 if not present
response = requests.get(download_url, stream=True, timeout=60)
response.raise_for_status()
file_size = int(response.headers.get("Content-Length", 0))
# Check if file exists and has the same size
# Check if file exists and has same size
if os.path.exists(destination):
file_size_local = os.path.getsize(destination)
if file_size == file_size_local:
if file_size and file_size == file_size_local:
print(f"File already exists and is up-to-date: {destination}")
return True # Indicate success without re-downloading
return True
block_size = 1024 # 1 Kilobyte
# Initialize the progress bar with total file size
progress_bar_description = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=progress_bar_description) as progress_bar:
block_size = 1024 # 1 KB
desc = os.path.basename(download_url)
with tqdm(total=file_size, unit="iB", unit_scale=True, desc=desc) as progress_bar:
with open(destination, "wb") as file:
while True:
chunk = response.read(block_size)
if not chunk:
break
for chunk in response.iter_content(chunk_size=block_size):
if chunk:
file.write(chunk)
progress_bar.update(len(chunk))
return True
@@ -76,13 +71,13 @@ def download_file(url, destination, backup_url=None):
try:
if _attempt_download(url):
return
except (urllib.error.HTTPError, urllib.error.URLError):
except requests.exceptions.RequestException:
if backup_url is not None:
print(f"Primary URL ({url}) failed. Attempting backup URL: {backup_url}")
try:
if _attempt_download(backup_url):
return
except urllib.error.HTTPError:
except requests.exceptions.RequestException:
pass
# If we reach here, both attempts have failed