Switch from urllib to requests to improve reliability (#867)

* Switch from urllib to requests to improve reliability

* Keep ruff linter-specific

* update

* update

* update

pkg/llms_from_scratch/ch05.py

@@ -7,11 +7,11 @@ from .ch04 import generate_text_simple
 
 import json
 import os
-import urllib.request
 
 import numpy as np
 import matplotlib.pyplot as plt
 from matplotlib.ticker import MaxNLocator
+import requests
 import torch
 from tqdm import tqdm
 
@@ -279,44 +279,40 @@ def download_and_load_gpt2(model_size, models_dir):
 
 def download_file(url, destination, backup_url=None):
     def _attempt_download(download_url):
-        with urllib.request.urlopen(download_url) as response:
-            # Get the total file size from headers, defaulting to 0 if not present
-            file_size = int(response.headers.get("Content-Length", 0))
+        response = requests.get(download_url, stream=True, timeout=60)
+        response.raise_for_status()
 
-            # Check if file exists and has the same size
-            if os.path.exists(destination):
-                file_size_local = os.path.getsize(destination)
-                if file_size == file_size_local:
-                    print(f"File already exists and is up-to-date: {destination}")
-                    return True  # Indicate success without re-downloading
+        file_size = int(response.headers.get("Content-Length", 0))
 
-            block_size = 1024  # 1 Kilobyte
+        # Check if file exists and has same size
+        if os.path.exists(destination):
+            file_size_local = os.path.getsize(destination)
+            if file_size and file_size == file_size_local:
+                print(f"File already exists and is up-to-date: {destination}")
+                return True
 
-            # Initialize the progress bar with total file size
-            progress_bar_description = os.path.basename(download_url)
-            with tqdm(total=file_size, unit="iB", unit_scale=True, desc=progress_bar_description) as progress_bar:
-                with open(destination, "wb") as file:
-                    while True:
-                        chunk = response.read(block_size)
-                        if not chunk:
-                            break
+        block_size = 1024  # 1 KB
+        desc = os.path.basename(download_url)
+        with tqdm(total=file_size, unit="iB", unit_scale=True, desc=desc) as progress_bar:
+            with open(destination, "wb") as file:
+                for chunk in response.iter_content(chunk_size=block_size):
+                    if chunk:
                         file.write(chunk)
                         progress_bar.update(len(chunk))
-            return True
+        return True
 
     try:
         if _attempt_download(url):
             return
-    except (urllib.error.HTTPError, urllib.error.URLError):
+    except requests.exceptions.RequestException:
         if backup_url is not None:
             print(f"Primary URL ({url}) failed. Attempting backup URL: {backup_url}")
             try:
                 if _attempt_download(backup_url):
                     return
-            except urllib.error.HTTPError:
+            except requests.exceptions.RequestException:
                 pass
 
-        # If we reach here, both attempts have failed
         error_message = (
             f"Failed to download from both primary URL ({url})"
             f"{' and backup URL (' + backup_url + ')' if backup_url else ''}."

pkg/llms_from_scratch/ch06.py

@@ -4,11 +4,11 @@
 # Code: https://github.com/rasbt/LLMs-from-scratch
 
 
-import urllib.request
 import zipfile
 import os
 from pathlib import Path
 
+import requests
 import matplotlib.pyplot as plt
 from torch.utils.data import Dataset
 import torch
@@ -21,9 +21,12 @@ def download_and_unzip_spam_data(url, zip_path, extracted_path, data_file_path):
         return
 
     # Downloading the file
-    with urllib.request.urlopen(url) as response:
-        with open(zip_path, "wb") as out_file:
-            out_file.write(response.read())
+    response = requests.get(url, stream=True, timeout=60)
+    response.raise_for_status()
+    with open(zip_path, "wb") as out_file:
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                out_file.write(chunk)
 
     # Unzipping the file
     with zipfile.ZipFile(zip_path, "r") as zip_ref:

pkg/llms_from_scratch/ch07.py

@@ -6,7 +6,7 @@
 import json
 import os
 import psutil
-import urllib
+import requests
 
 import torch
 from tqdm import tqdm
@@ -14,24 +14,46 @@ from torch.utils.data import Dataset
 
 
 def download_and_load_file(file_path, url):
-
     if not os.path.exists(file_path):
-        with urllib.request.urlopen(url) as response:
-            text_data = response.read().decode("utf-8")
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+        text_data = response.text
         with open(file_path, "w", encoding="utf-8") as file:
             file.write(text_data)
 
-    # The book originally contained this unnecessary "else" clause:
-    # else:
-    #     with open(file_path, "r", encoding="utf-8") as file:
-    #         text_data = file.read()
-
     with open(file_path, "r", encoding="utf-8") as file:
         data = json.load(file)
 
     return data
 
 
+# The book originally used the following code below
+# However, urllib uses older protocol settings that
+# can cause problems for some readers using a VPN.
+# The `requests` version above is more robust
+# in that regard.
+
+
+# import urllib
+
+# def download_and_load_file(file_path, url):
+
+#     if not os.path.exists(file_path):
+#         with urllib.request.urlopen(url) as response:
+#             text_data = response.read().decode("utf-8")
+#         with open(file_path, "w", encoding="utf-8") as file:
+#             file.write(text_data)
+
+#     else:
+#         with open(file_path, "r", encoding="utf-8") as file:
+#             text_data = file.read()
+
+#     with open(file_path, "r", encoding="utf-8") as file:
+#         data = json.load(file)
+
+#     return data
+
+
 def format_input(entry):
     instruction_text = (
         f"Below is an instruction that describes a task. "
@@ -202,27 +224,16 @@ def query_model(
         }
     }
 
-    # Convert the dictionary to a JSON formatted string and encode it to bytes
-    payload = json.dumps(data).encode("utf-8")
-
-    # Create a request object, setting the method to POST and adding necessary headers
-    request = urllib.request.Request(
-        url,
-        data=payload,
-        method="POST"
-    )
-    request.add_header("Content-Type", "application/json")
-
-    # Send the request and capture the response
-    response_data = ""
-    with urllib.request.urlopen(request) as response:
-        # Read and decode the response
-        while True:
-            line = response.readline().decode("utf-8")
+    # Send the POST request
+    with requests.post(url, json=data, stream=True, timeout=30) as r:
+        r.raise_for_status()
+        response_data = ""
+        for line in r.iter_lines(decode_unicode=True):
             if not line:
-                break
+                continue
             response_json = json.loads(line)
-            response_data += response_json["message"]["content"]
+            if "message" in response_json:
+                response_data += response_json["message"]["content"]
 
     return response_data
 

pkg/llms_from_scratch/qwen3.py

@@ -6,9 +6,9 @@
 import os
 import json
 import re
-import urllib.request
 from pathlib import Path
 
+import requests
 import torch
 import torch.nn as nn
 
@@ -660,7 +660,12 @@ def download_from_huggingface(repo_id, filename, local_dir, revision="main"):
         print(f"File already exists: {dest_path}")
     else:
         print(f"Downloading {url} to {dest_path}...")
-        urllib.request.urlretrieve(url, dest_path)
+        response = requests.get(url, stream=True, timeout=60)
+        response.raise_for_status()
+        with open(dest_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
 
     return dest_path
 

pkg/llms_from_scratch/tests/test_appendix_e.py

@@ -12,9 +12,9 @@ from llms_from_scratch.ch06 import (
 from llms_from_scratch.appendix_e import replace_linear_with_lora
 
 from pathlib import Path
-import urllib
 
 import pandas as pd
+import requests
 import tiktoken
 import torch
 from torch.utils.data import DataLoader, Subset
@@ -35,7 +35,7 @@ def test_train_classifier_lora(tmp_path):
         download_and_unzip_spam_data(
             url, zip_path, extracted_path, data_file_path
         )
-    except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as e:
+    except (requests.exceptions.RequestException, TimeoutError) as e:
         print(f"Primary URL failed: {e}. Trying backup URL...")
         backup_url = "https://f001.backblazeb2.com/file/LLMs-from-scratch/sms%2Bspam%2Bcollection.zip"
         download_and_unzip_spam_data(

pkg/llms_from_scratch/tests/test_ch02.py

@@ -6,8 +6,8 @@
 from llms_from_scratch.ch02 import create_dataloader_v1
 
 import os
-import urllib.request
 
+import requests
 import pytest
 import torch
 
@@ -16,11 +16,17 @@ import torch
 def test_dataloader(tmp_path, file_name):
 
     if not os.path.exists("the-verdict.txt"):
-        url = ("https://raw.githubusercontent.com/rasbt/"
-               "LLMs-from-scratch/main/ch02/01_main-chapter-code/"
-               "the-verdict.txt")
+        url = (
+            "https://raw.githubusercontent.com/rasbt/"
+            "LLMs-from-scratch/main/ch02/01_main-chapter-code/"
+            "the-verdict.txt"
+        )
         file_path = "the-verdict.txt"
-        urllib.request.urlretrieve(url, file_path)
+
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+        with open(file_path, "wb") as f:
+            f.write(response.content)
 
     with open("the-verdict.txt", "r", encoding="utf-8") as f:
         raw_text = f.read()

pkg/llms_from_scratch/tests/test_ch05.py

@@ -8,8 +8,8 @@ from llms_from_scratch.ch04 import GPTModel, GPTModelFast
 from llms_from_scratch.ch05 import train_model_simple
 
 import os
-import urllib
 
+import requests
 import pytest
 import tiktoken
 import torch
@@ -46,8 +46,9 @@ def test_train_simple(tmp_path, ModelClass):
     url = "https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/main/ch02/01_main-chapter-code/the-verdict.txt"
 
     if not os.path.exists(file_path):
-        with urllib.request.urlopen(url) as response:
-            text_data = response.read().decode("utf-8")
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+        text_data = response.text
         with open(file_path, "w", encoding="utf-8") as f:
             f.write(text_data)
     else:

pkg/llms_from_scratch/tests/test_ch06.py

@@ -11,8 +11,8 @@ from llms_from_scratch.ch06 import (
 )
 
 from pathlib import Path
-import urllib
 
+import requests
 import pandas as pd
 import tiktoken
 import torch
@@ -34,7 +34,7 @@ def test_train_classifier(tmp_path):
         download_and_unzip_spam_data(
             url, zip_path, extracted_path, data_file_path
         )
-    except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as e:
+    except (requests.exceptions.RequestException, TimeoutError) as e:
         print(f"Primary URL failed: {e}. Trying backup URL...")
         backup_url = "https://f001.backblazeb2.com/file/LLMs-from-scratch/sms%2Bspam%2Bcollection.zip"
         download_and_unzip_spam_data(

pkg/llms_from_scratch/utils.py

@@ -9,10 +9,9 @@ import ast
 import re
 import types
 from pathlib import Path
-import urllib.request
-import urllib.parse
 
 import nbformat
+import requests
 
 
 def _extract_imports(src: str):
@@ -125,21 +124,24 @@ def import_definitions_from_notebook(nb_dir_or_path, notebook_name=None, *, extr
     exec(src, mod.__dict__)
     return mod
 
+
 def download_file(url, out_dir="."):
     """Simple file download utility for tests."""
-    from pathlib import Path
     out_dir = Path(out_dir)
     out_dir.mkdir(parents=True, exist_ok=True)
-    filename = Path(urllib.parse.urlparse(url).path).name
+    filename = Path(url).name
     dest = out_dir / filename
-    
+
     if dest.exists():
         return dest
-        
+
     try:
-        with urllib.request.urlopen(url) as response:
-            with open(dest, 'wb') as f:
-                f.write(response.read())
+        response = requests.get(url, stream=True, timeout=30)
+        response.raise_for_status()
+        with open(dest, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
         return dest
     except Exception as e:
         raise RuntimeError(f"Failed to download {url}: {e}")