append_to_parquet(df: DataFrame, base_path: str | Path, *, partition_cols: list[str] | tuple[str, ...] = ('system_id', 'date')) -> None
Append a snapshot to a Hive-partitioned Parquet dataset (append-only, fast).
A date column (UTC, YYYY-MM-DD) is derived from fetched_at when missing,
for stable daily partitioning. Each call writes a new file (unique basename), so
concurrent pollers never clobber each other; de-duplication happens on read, not write.
See Also
generate_manifest : Manifest of the written archive.
build_availability_panel : Read the archive back into a panel.
Source code in src/gbfs_toolkit/io/timeseries.py
| def append_to_parquet(
df: pd.DataFrame,
base_path: str | Path,
*,
partition_cols: list[str] | tuple[str, ...] = ("system_id", "date"),
) -> None:
"""Append a snapshot to a Hive-partitioned Parquet dataset (append-only, fast).
A ``date`` column (UTC, ``YYYY-MM-DD``) is derived from ``fetched_at`` when missing,
for stable daily partitioning. Each call writes a *new* file (unique basename), so
concurrent pollers never clobber each other; de-duplication happens on read, not write.
See Also
--------
[`generate_manifest`][gbfs_toolkit.generate_manifest] : Manifest of the written archive.
[`build_availability_panel`][gbfs_toolkit.build_availability_panel] : Read the archive back into a panel.
"""
ds = _require_pyarrow()
import pyarrow as pa
out = df.copy()
if "date" in partition_cols and "date" not in out.columns:
out["date"] = pd.to_datetime(out["fetched_at"], utc=True).dt.strftime("%Y-%m-%d")
table = pa.Table.from_pandas(out, preserve_index=False)
ds.write_dataset(
table,
base_path,
format="parquet",
partitioning=list(partition_cols),
partitioning_flavor="hive",
basename_template=f"part-{uuid.uuid4().hex}-{{i}}.parquet",
existing_data_behavior="overwrite_or_ignore",
)
|