Skip to content

fetch_multiple

fetch_multiple

fetch_multiple(system_ids: list[str], *, catalog: DataFrame | None = None, max_workers: int = 5, session: Any = None, progress: bool = False, **kwargs: Any) -> dict[str, GBFSFeed | Exception]

Resolve and open many systems concurrently (threaded) for comparative studies.

Returns {system_id: GBFSFeed}, or the Exception for systems that failed, so one dead feed never sinks a 50-city pull. Discovery runs eagerly so failures surface here; data fetches stay lazy on each returned feed.

Pass a shared requests.Session to pool connections across all systems (strongly recommended for repeated polling, avoids TCP/port exhaustion). Ignored if you also pass your own get_json.

Set progress=True for feedback on a long pull: a tqdm bar when tqdm is installed (pip install gbfs-toolkit[cli]), otherwise periodic log lines. Feeds complete out of order; a failure is logged and recorded, never raised, so the bar always finishes.

See Also

fetch_feed_json : Fetch one document. systems_catalog : Resolve the system ids to fetch.

Source code in src/gbfs_toolkit/io/fetch.py
def fetch_multiple(
    system_ids: list[str],
    *,
    catalog: pd.DataFrame | None = None,
    max_workers: int = 5,
    session: Any = None,
    progress: bool = False,
    **kwargs: Any,
) -> dict[str, GBFSFeed | Exception]:
    """Resolve and open many systems concurrently (threaded) for comparative studies.

    Returns ``{system_id: GBFSFeed}``, or the ``Exception`` for systems that failed,
    so one dead feed never sinks a 50-city pull. Discovery runs eagerly so failures
    surface here; data fetches stay lazy on each returned feed.

    Pass a shared ``requests.Session`` to pool connections across all systems (strongly
    recommended for repeated polling, avoids TCP/port exhaustion). Ignored if you also
    pass your own ``get_json``.

    Set ``progress=True`` for feedback on a long pull: a tqdm bar when tqdm is installed
    (``pip install gbfs-toolkit[cli]``), otherwise periodic log lines. Feeds complete out
    of order; a failure is logged and recorded, never raised, so the bar always finishes.

    See Also
    --------
    [`fetch_feed_json`][gbfs_toolkit.fetch_feed_json] : Fetch one document.
    [`systems_catalog`][gbfs_toolkit.systems_catalog] : Resolve the system ids to fetch.
    """
    from concurrent.futures import ThreadPoolExecutor, as_completed

    from gbfs_toolkit.io.catalog import systems_catalog

    cat = catalog if catalog is not None else systems_catalog()
    if "get_json" not in kwargs:
        # Default to a pooled, retry/backoff session so one flaky feed never sinks the batch.
        session = session if session is not None else build_session()
        kwargs = {**kwargs, "get_json": _session_getter(session, timeout=kwargs.get("timeout", 30))}

    def _open(sid: str) -> GBFSFeed:
        feed = GBFSFeed.from_system_id(sid, catalog=cat, **kwargs)
        _ = feed.feeds  # force discovery so a broken gbfs.json fails now
        return feed

    results: dict[str, GBFSFeed | Exception] = {}
    with ThreadPoolExecutor(max_workers=max_workers) as pool:
        futures = {pool.submit(_open, sid): sid for sid in system_ids}
        done = as_completed(futures)
        if progress:
            done = _with_progress(done, total=len(futures), desc="fetching feeds")
        for fut in done:
            sid = futures[fut]
            try:
                results[sid] = fut.result()
            except Exception as exc:  # noqa: BLE001 (we deliberately capture per-system)
                results[sid] = exc
                _log.warning("fetch_multiple: %s failed (%s)", sid, exc)
    return results