Skip to content

build_availability_panel

build_availability_panel

build_availability_panel(base_path: str | Path, *, system_id: str | None = None, start_time: str | Timestamp | None = None, end_time: str | Timestamp | None = None, resample_freq: str | None = None, dedup: bool = True, columns: list[str] | None = None, filters: Any = None, target_tz: str | None = None) -> DataFrame

Read a partitioned dataset into a tidy availability panel.

Filters by system_id and the date partition before loading (memory-bounded), then de-duplicates redundant polls (same station_id + last_reported) and optionally resamples each station to a fixed frequency (forward-filled).

Scaling to large lakes

For multi-month / multi-city panels that would not fit in memory, push the work down into PyArrow:

  • columns: project only the columns you need (the keys system_id, station_id, fetched_at and, when de-duplicating, last_reported are always read). Fewer columns ⇒ less I/O and RAM.
  • filters: an extra pyarrow.dataset predicate ANDed with the built-in system/date filter, applied before materialising (row-group pruning). Build it with import pyarrow.dataset as ds; ds.field("num_bikes_available") == 0.
Local time

Pass target_tz (e.g. "America/Los_Angeles") to convert fetched_at / last_reported to that zone before any dedup or resample. Resampling tz-aware UTC data would otherwise cut "days" at UTC midnight (i.e. mid-afternoon local time), silently corrupting diurnal/daily aggregations.

Returns:

Type Description
DataFrame

MultiIndexed by (system_id, station_id, fetched_at), sorted.

See Also

availability : A single-snapshot availability frame. calculate_net_flow : Per-interval flow from this panel. stockout_episodes : Outage episodes from this panel.

Source code in src/gbfs_toolkit/io/timeseries.py
def build_availability_panel(
    base_path: str | Path,
    *,
    system_id: str | None = None,
    start_time: str | pd.Timestamp | None = None,
    end_time: str | pd.Timestamp | None = None,
    resample_freq: str | None = None,
    dedup: bool = True,
    columns: list[str] | None = None,
    filters: Any = None,
    target_tz: str | None = None,
) -> pd.DataFrame:
    """Read a partitioned dataset into a tidy availability panel.

    Filters by ``system_id`` and the ``date`` partition *before* loading (memory-bounded),
    then de-duplicates redundant polls (same ``station_id`` + ``last_reported``) and
    optionally resamples each station to a fixed frequency (forward-filled).

    Scaling to large lakes
    ----------------------
    For multi-month / multi-city panels that would not fit in memory, push the work down
    into PyArrow:

    - ``columns``: project only the columns you need (the keys ``system_id``,
      ``station_id``, ``fetched_at`` and, when de-duplicating, ``last_reported`` are always
      read). Fewer columns ⇒ less I/O and RAM.
    - ``filters``: an extra ``pyarrow.dataset`` predicate ANDed with the built-in
      system/date filter, applied *before* materialising (row-group pruning). Build it with
      ``import pyarrow.dataset as ds; ds.field("num_bikes_available") == 0``.

    Local time
    ----------
    Pass ``target_tz`` (e.g. ``"America/Los_Angeles"``) to convert ``fetched_at`` /
    ``last_reported`` to that zone **before** any dedup or resample. Resampling tz-aware UTC
    data would otherwise cut "days" at UTC midnight (i.e. mid-afternoon local time),
    silently corrupting diurnal/daily aggregations.

    Returns
    -------
    pandas.DataFrame
        MultiIndexed by ``(system_id, station_id, fetched_at)``, sorted.

    See Also
    --------
    [`availability`][gbfs_toolkit.availability] : A single-snapshot availability frame.
    [`calculate_net_flow`][gbfs_toolkit.calculate_net_flow] : Per-interval flow from this panel.
    [`stockout_episodes`][gbfs_toolkit.stockout_episodes] : Outage episodes from this panel.
    """
    ds = _require_pyarrow()
    dataset = ds.dataset(str(base_path), format="parquet", partitioning="hive")

    start = _as_utc(start_time) if start_time is not None else None
    end = _as_utc(end_time) if end_time is not None else None

    filt = None
    if system_id is not None:
        filt = ds.field("system_id") == system_id
    if start is not None and "date" in dataset.schema.names:
        cond = ds.field("date") >= start.strftime("%Y-%m-%d")
        filt = cond if filt is None else filt & cond
    if end is not None and "date" in dataset.schema.names:
        cond = ds.field("date") <= end.strftime("%Y-%m-%d")
        filt = cond if filt is None else filt & cond
    if filters is not None:
        filt = filters if filt is None else filt & filters

    read_cols = None
    if columns is not None:
        keys = ["system_id", "station_id", "fetched_at"]
        if dedup:
            keys.append("last_reported")
        wanted = list(dict.fromkeys([*keys, *columns]))
        read_cols = [c for c in wanted if c in dataset.schema.names]

    df = dataset.to_table(filter=filt, columns=read_cols).to_pandas()
    if df.empty:
        return df

    df["fetched_at"] = pd.to_datetime(df["fetched_at"], utc=True)
    if "last_reported" in df:
        df["last_reported"] = pd.to_datetime(df["last_reported"], utc=True)
    if target_tz is not None:
        # Convert to local time *before* dedup/resample so daily boundaries are local midnight.
        df["fetched_at"] = df["fetched_at"].dt.tz_convert(target_tz)
        if "last_reported" in df:
            df["last_reported"] = df["last_reported"].dt.tz_convert(target_tz)
    df = df.sort_values(["system_id", "station_id", "fetched_at"])

    if dedup and "last_reported" in df:
        df = df.drop_duplicates(subset=["system_id", "station_id", "last_reported"], keep="last")
    if start is not None:
        df = df[df["fetched_at"] >= start]
    if end is not None:
        df = df[df["fetched_at"] <= end]

    if resample_freq:
        cols = [c for c in ("num_bikes_available", "num_docks_available") if c in df]
        parts = []
        for (sid, stid), g in df.groupby(["system_id", "station_id"], sort=False):
            r = g.set_index("fetched_at")[cols].resample(resample_freq).ffill()
            r["system_id"], r["station_id"] = sid, stid
            parts.append(r.reset_index())
        df = pd.concat(parts, ignore_index=True)

    return df.set_index(["system_id", "station_id", "fetched_at"]).sort_index()