fetch_multiple(system_ids: list[str], *, catalog: DataFrame | None = None, max_workers: int = 5, session: Any = None, progress: bool = False, **kwargs: Any) -> dict[str, GBFSFeed | Exception]
Resolve and open many systems concurrently (threaded) for comparative studies.
Returns {system_id: GBFSFeed}, or the Exception for systems that failed,
so one dead feed never sinks a 50-city pull. Discovery runs eagerly so failures
surface here; data fetches stay lazy on each returned feed.
Pass a shared requests.Session to pool connections across all systems (strongly
recommended for repeated polling, avoids TCP/port exhaustion). Ignored if you also
pass your own get_json.
Set progress=True for feedback on a long pull: a tqdm bar when tqdm is installed
(pip install gbfs-toolkit[cli]), otherwise periodic log lines. Feeds complete out
of order; a failure is logged and recorded, never raised, so the bar always finishes.
See Also
fetch_feed_json : Fetch one document.
systems_catalog : Resolve the system ids to fetch.
Source code in src/gbfs_toolkit/io/fetch.py
| def fetch_multiple(
system_ids: list[str],
*,
catalog: pd.DataFrame | None = None,
max_workers: int = 5,
session: Any = None,
progress: bool = False,
**kwargs: Any,
) -> dict[str, GBFSFeed | Exception]:
"""Resolve and open many systems concurrently (threaded) for comparative studies.
Returns ``{system_id: GBFSFeed}``, or the ``Exception`` for systems that failed,
so one dead feed never sinks a 50-city pull. Discovery runs eagerly so failures
surface here; data fetches stay lazy on each returned feed.
Pass a shared ``requests.Session`` to pool connections across all systems (strongly
recommended for repeated polling, avoids TCP/port exhaustion). Ignored if you also
pass your own ``get_json``.
Set ``progress=True`` for feedback on a long pull: a tqdm bar when tqdm is installed
(``pip install gbfs-toolkit[cli]``), otherwise periodic log lines. Feeds complete out
of order; a failure is logged and recorded, never raised, so the bar always finishes.
See Also
--------
[`fetch_feed_json`][gbfs_toolkit.fetch_feed_json] : Fetch one document.
[`systems_catalog`][gbfs_toolkit.systems_catalog] : Resolve the system ids to fetch.
"""
from concurrent.futures import ThreadPoolExecutor, as_completed
from gbfs_toolkit.io.catalog import systems_catalog
cat = catalog if catalog is not None else systems_catalog()
if "get_json" not in kwargs:
# Default to a pooled, retry/backoff session so one flaky feed never sinks the batch.
session = session if session is not None else build_session()
kwargs = {**kwargs, "get_json": _session_getter(session, timeout=kwargs.get("timeout", 30))}
def _open(sid: str) -> GBFSFeed:
feed = GBFSFeed.from_system_id(sid, catalog=cat, **kwargs)
_ = feed.feeds # force discovery so a broken gbfs.json fails now
return feed
results: dict[str, GBFSFeed | Exception] = {}
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = {pool.submit(_open, sid): sid for sid in system_ids}
done = as_completed(futures)
if progress:
done = _with_progress(done, total=len(futures), desc="fetching feeds")
for fut in done:
sid = futures[fut]
try:
results[sid] = fut.result()
except Exception as exc: # noqa: BLE001 (we deliberately capture per-system)
results[sid] = exc
_log.warning("fetch_multiple: %s failed (%s)", sid, exc)
return results
|