mirror of
https://github.com/rasbt/LLMs-from-scratch.git
synced 2026-04-10 12:33:42 +00:00
harded the link checker
This commit is contained in:
46
conftest.py
46
conftest.py
@@ -1,17 +1,57 @@
|
|||||||
import os
|
import os
|
||||||
|
import time
|
||||||
|
from functools import wraps
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
def _get_env_number(name, default, cast):
|
||||||
|
value = os.environ.get(name)
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
|
||||||
|
try:
|
||||||
|
return cast(value)
|
||||||
|
except ValueError:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
def pytest_configure(config):
|
def pytest_configure(config):
|
||||||
if not getattr(config.option, "check_links", False):
|
if not getattr(config.option, "check_links", False):
|
||||||
return
|
return
|
||||||
|
|
||||||
timeout = float(os.environ.get("CHECK_LINKS_TIMEOUT", "10"))
|
timeout = _get_env_number("CHECK_LINKS_TIMEOUT", 10.0, float)
|
||||||
original_request = requests.sessions.Session.request
|
max_retries = max(0, _get_env_number("CHECK_LINKS_RETRIES", 2, int))
|
||||||
|
retry_backoff = max(0.0, _get_env_number("CHECK_LINKS_RETRY_BACKOFF", 1.0, float))
|
||||||
|
current_request = requests.sessions.Session.request
|
||||||
|
|
||||||
|
if getattr(current_request, "_check_links_wrapped", False):
|
||||||
|
return
|
||||||
|
|
||||||
|
retryable_methods = {"GET", "HEAD"}
|
||||||
|
retryable_errors = (
|
||||||
|
requests.exceptions.Timeout,
|
||||||
|
requests.exceptions.ConnectionError,
|
||||||
|
)
|
||||||
|
|
||||||
|
@wraps(current_request)
|
||||||
def request_with_timeout(self, method, url, **kwargs):
|
def request_with_timeout(self, method, url, **kwargs):
|
||||||
if kwargs.get("timeout") is None:
|
if kwargs.get("timeout") is None:
|
||||||
kwargs["timeout"] = timeout
|
kwargs["timeout"] = timeout
|
||||||
return original_request(self, method, url, **kwargs)
|
|
||||||
|
|
||||||
|
method_name = (method or "").upper()
|
||||||
|
|
||||||
|
for attempt in range(max_retries + 1):
|
||||||
|
try:
|
||||||
|
return current_request(self, method, url, **kwargs)
|
||||||
|
except retryable_errors:
|
||||||
|
should_retry = method_name in retryable_methods and attempt < max_retries
|
||||||
|
if not should_retry:
|
||||||
|
raise
|
||||||
|
|
||||||
|
# Retries smooth over transient CI/network blips without masking real 4xx/5xx failures.
|
||||||
|
if retry_backoff:
|
||||||
|
time.sleep(retry_backoff * (attempt + 1))
|
||||||
|
|
||||||
|
request_with_timeout._check_links_wrapped = True
|
||||||
requests.sessions.Session.request = request_with_timeout
|
requests.sessions.Session.request = request_with_timeout
|
||||||
|
|||||||
Reference in New Issue
Block a user