books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch from urllib to requests to improve reliability (#867)

Browse Source 
* Switch from urllib to requests to improve reliability

* Keep ruff linter-specific

* update

* update

* update
This commit is contained in:
Sebastian Raschka
2025-10-07 15:22:59 -05:00
committed by GitHub
parent 8552565bda
commit 7bd263144e
47 changed files with 592 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
ch07/01_main-chapter-code/ch07.ipynb
View File

@@ -169,10 +169,33 @@
"source": [
"import json\n",
"import os\n",
"import urllib\n",
"import requests\n",
"\n",
"\n",
"def download_and_load_file(file_path, url):\n",
" if not os.path.exists(file_path):\n",
" response = requests.get(url, timeout=30)\n",
" response.raise_for_status()\n",
" text_data = response.text\n",
" with open(file_path, \"w\", encoding=\"utf-8\") as file:\n",
" file.write(text_data)\n",
"\n",
" with open(file_path, \"r\", encoding=\"utf-8\") as file:\n",
" data = json.load(file)\n",
"\n",
" return data\n",
"\n",
"\n",
"# The book originally used the following code below\n",
"# However, urllib uses older protocol settings that\n",
"# can cause problems for some readers using a VPN.\n",
"# The `requests` version above is more robust\n",
"# in that regard.\n",
"\n",
"\"\"\"\n",
"import urllib\n",
"\n",
"def download_and_load_file(file_path, url):\n",
"\n",
" if not os.path.exists(file_path):\n",
" with urllib.request.urlopen(url) as response:\n",
@@ -180,15 +203,15 @@
" with open(file_path, \"w\", encoding=\"utf-8\") as file:\n",
" file.write(text_data)\n",
"\n",
" # The book originally contained this unnecessary \"else\" clause:\n",
" #else:\n",
" # with open(file_path, \"r\", encoding=\"utf-8\") as file:\n",
" # text_data = file.read()\n",
" else:\n",
" with open(file_path, \"r\", encoding=\"utf-8\") as file:\n",
" text_data = file.read()\n",
"\n",
" with open(file_path, \"r\", encoding=\"utf-8\") as file:\n",
" data = json.load(file)\n",
"\n",
" return data\n",
"\"\"\"\n",
"\n",
"\n",
"file_path = \"instruction-data.json\"\n",
@@ -2490,7 +2513,8 @@
}
],
"source": [
"import urllib.request\n",
"import requests # noqa: F811\n",
"# import urllib.request\n",
"\n",
"def query_model(\n",
" prompt,\n",
@@ -2512,7 +2536,8 @@
" }\n",
" }\n",
"\n",
"\n",
" \n",
" \"\"\"\n",
" # Convert the dictionary to a JSON formatted string and encode it to bytes\n",
" payload = json.dumps(data).encode(\"utf-8\")\n",
"\n",
@@ -2536,6 +2561,26 @@
" response_data += response_json[\"message\"][\"content\"]\n",
"\n",
" return response_data\n",
" \"\"\"\n",
"\n",
" # The book originally used the commented-out above, which is based\n",
" # on urllib. It works generally fine, but some readers reported\n",
" # issues with using urlib when using a (company) VPN.\n",
" # The code below uses the requests library, which doesn't seem\n",
" # to have these issues.\n",
"\n",
" # Send the POST request\n",
" with requests.post(url, json=data, stream=True, timeout=30) as r:\n",
" r.raise_for_status()\n",
" response_data = \"\"\n",
" for line in r.iter_lines(decode_unicode=True):\n",
" if not line:\n",
" continue\n",
" response_json = json.loads(line)\n",
" if \"message\" in response_json:\n",
" response_data += response_json[\"message\"][\"content\"]\n",
"\n",
" return response_data\n",
"\n",
"\n",
"model = \"llama3\"\n",