Skip to content

fetch_feed_json

fetch_feed_json

fetch_feed_json(url: str, *, session: Any = None, etag: str | None = None, last_modified: str | None = None, timeout: int = 30) -> FeedResponse

Conditionally fetch a feed, honouring HTTP caching: the polite way to poll.

Pass the etag / last_modified returned by the previous call; if the server replies 304 Not Modified, this raises GBFSNotModified so your scraper can skip re-ingesting an unchanged snapshot (saving bandwidth and avoiding an IP ban). Otherwise returns a FeedResponse (data, etag, last_modified) to store for next time. Requires the [fetch] extra.

See Also

GBFSFeed : The high-level feed interface. fetch_multiple : Fetch many systems concurrently. build_session : Configure the HTTP session.

Source code in src/gbfs_toolkit/io/fetch.py
def fetch_feed_json(
    url: str,
    *,
    session: Any = None,
    etag: str | None = None,
    last_modified: str | None = None,
    timeout: int = 30,
) -> FeedResponse:
    """Conditionally fetch a feed, honouring HTTP caching: the polite way to poll.

    Pass the ``etag`` / ``last_modified`` returned by the previous call; if the server replies
    **304 Not Modified**, this raises [`GBFSNotModified`][gbfs_toolkit.GBFSNotModified] so your
    scraper can skip re-ingesting an unchanged snapshot (saving bandwidth and avoiding an
    IP ban). Otherwise returns a [`FeedResponse`][gbfs_toolkit.FeedResponse] ``(data, etag, last_modified)`` to store
    for next time. Requires the ``[fetch]`` extra.

    See Also
    --------
    [`GBFSFeed`][gbfs_toolkit.GBFSFeed] : The high-level feed interface.
    [`fetch_multiple`][gbfs_toolkit.fetch_multiple] : Fetch many systems concurrently.
    [`build_session`][gbfs_toolkit.build_session] : Configure the HTTP session.
    """
    requests = _require_requests()
    headers = {"User-Agent": _USER_AGENT}
    if etag:
        headers["If-None-Match"] = etag
    if last_modified:
        headers["If-Modified-Since"] = last_modified
    getter = session if session is not None else requests
    try:
        resp = getter.get(url, timeout=timeout, headers=headers)
    except requests.RequestException as e:
        raise GBFSFetchError(f"failed to fetch {url}: {e}") from e
    if resp.status_code == 304:
        raise GBFSNotModified(url)
    try:
        resp.raise_for_status()  # 4xx/5xx
        payload = resp.json()  # malformed body
    except (requests.HTTPError, ValueError) as e:
        raise GBFSFetchError(f"failed to fetch {url}: {e}") from e
    return FeedResponse(payload, resp.headers.get("ETag"), resp.headers.get("Last-Modified"))