fetch_feed_json(url: str, *, session: Any = None, etag: str | None = None, last_modified: str | None = None, timeout: int = 30) -> FeedResponse
Conditionally fetch a feed, honouring HTTP caching: the polite way to poll.
Pass the etag / last_modified returned by the previous call; if the server replies
304 Not Modified, this raises GBFSNotModified so your
scraper can skip re-ingesting an unchanged snapshot (saving bandwidth and avoiding an
IP ban). Otherwise returns a FeedResponse (data, etag, last_modified) to store
for next time. Requires the [fetch] extra.
See Also
GBFSFeed : The high-level feed interface.
fetch_multiple : Fetch many systems concurrently.
build_session : Configure the HTTP session.
Source code in src/gbfs_toolkit/io/fetch.py
| def fetch_feed_json(
url: str,
*,
session: Any = None,
etag: str | None = None,
last_modified: str | None = None,
timeout: int = 30,
) -> FeedResponse:
"""Conditionally fetch a feed, honouring HTTP caching: the polite way to poll.
Pass the ``etag`` / ``last_modified`` returned by the previous call; if the server replies
**304 Not Modified**, this raises [`GBFSNotModified`][gbfs_toolkit.GBFSNotModified] so your
scraper can skip re-ingesting an unchanged snapshot (saving bandwidth and avoiding an
IP ban). Otherwise returns a [`FeedResponse`][gbfs_toolkit.FeedResponse] ``(data, etag, last_modified)`` to store
for next time. Requires the ``[fetch]`` extra.
See Also
--------
[`GBFSFeed`][gbfs_toolkit.GBFSFeed] : The high-level feed interface.
[`fetch_multiple`][gbfs_toolkit.fetch_multiple] : Fetch many systems concurrently.
[`build_session`][gbfs_toolkit.build_session] : Configure the HTTP session.
"""
requests = _require_requests()
headers = {"User-Agent": _USER_AGENT}
if etag:
headers["If-None-Match"] = etag
if last_modified:
headers["If-Modified-Since"] = last_modified
getter = session if session is not None else requests
try:
resp = getter.get(url, timeout=timeout, headers=headers)
except requests.RequestException as e:
raise GBFSFetchError(f"failed to fetch {url}: {e}") from e
if resp.status_code == 304:
raise GBFSNotModified(url)
try:
resp.raise_for_status() # 4xx/5xx
payload = resp.json() # malformed body
except (requests.HTTPError, ValueError) as e:
raise GBFSFetchError(f"failed to fetch {url}: {e}") from e
return FeedResponse(payload, resp.headers.get("ETag"), resp.headers.get("Last-Modified"))
|