build_session(*, total: int = 3, backoff_factor: float = 0.5, status_forcelist: tuple[int, ...] = (429, 500, 502, 503, 504)) -> Any
A pooled requests.Session with polite retry/backoff: the right default for scraping.
Transient 429/5xx responses from operator API gateways are routine; this retries them with
exponential backoff instead of failing the whole poll. Reusing one session across systems
also pools TCP connections (avoids port exhaustion). Requires the [fetch] extra.
See Also
fetch_feed_json : The fetch that uses the session.
Source code in src/gbfs_toolkit/io/fetch.py
| def build_session(
*,
total: int = 3,
backoff_factor: float = 0.5,
status_forcelist: tuple[int, ...] = (429, 500, 502, 503, 504),
) -> Any:
"""A pooled ``requests.Session`` with polite retry/backoff: the right default for scraping.
Transient 429/5xx responses from operator API gateways are routine; this retries them with
exponential backoff instead of failing the whole poll. Reusing one session across systems
also pools TCP connections (avoids port exhaustion). Requires the ``[fetch]`` extra.
See Also
--------
[`fetch_feed_json`][gbfs_toolkit.fetch_feed_json] : The fetch that uses the session.
"""
requests = _require_requests()
from requests.adapters import HTTPAdapter
from urllib3.util import Retry
session = requests.Session()
retry = Retry(
total=total,
backoff_factor=backoff_factor,
status_forcelist=list(status_forcelist),
allowed_methods=frozenset({"GET", "HEAD"}),
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
session.headers.update({"User-Agent": _USER_AGENT})
return session
|