A cryptographic manifest of a Parquet lake, for citable, reproducible datasets.
Walks every *.parquet partition file, records its SHA-256 and size, and summarises
the dataset (systems, date span, row count). Drop the returned dict next to a Zenodo /
Dataverse deposit so a reviewer can verify byte-for-byte what was analysed.
Returns:
| Type |
Description |
dict
|
gbfs_toolkit_version, generated_at (UTC ISO), n_files, total_bytes,
total_rows, system_ids, min_date, max_date, and files (a sorted
list of {path, sha256, bytes} with paths relative to lake_dir).
|
See Also
append_to_parquet : The append step it documents.
coverage_report : Per-feed coverage summary.
Source code in src/gbfs_toolkit/io/timeseries.py
| def generate_manifest(lake_dir: str | Path, *, chunk_size: int = 1 << 20) -> dict:
"""A cryptographic manifest of a Parquet lake, for citable, reproducible datasets.
Walks every ``*.parquet`` partition file, records its SHA-256 and size, and summarises
the dataset (systems, date span, row count). Drop the returned dict next to a Zenodo /
Dataverse deposit so a reviewer can verify byte-for-byte what was analysed.
Returns
-------
dict
``gbfs_toolkit_version``, ``generated_at`` (UTC ISO), ``n_files``, ``total_bytes``,
``total_rows``, ``system_ids``, ``min_date``, ``max_date``, and ``files`` (a sorted
list of ``{path, sha256, bytes}`` with paths relative to ``lake_dir``).
See Also
--------
[`append_to_parquet`][gbfs_toolkit.append_to_parquet] : The append step it documents.
[`coverage_report`][gbfs_toolkit.coverage_report] : Per-feed coverage summary.
"""
from gbfs_toolkit import __version__
base = Path(lake_dir)
files = []
for p in sorted(base.rglob("*.parquet")):
h = hashlib.sha256()
with p.open("rb") as fh:
for block in iter(functools.partial(fh.read, chunk_size), b""):
h.update(block)
files.append(
{"path": str(p.relative_to(base)), "sha256": h.hexdigest(), "bytes": p.stat().st_size}
)
manifest: dict[str, Any] = {
"gbfs_toolkit_version": __version__,
"generated_at": pd.Timestamp.now(tz="UTC").isoformat(),
"n_files": len(files),
"total_bytes": sum(f["bytes"] for f in files),
"files": files,
}
try: # dataset-level summary (best-effort; needs a readable dataset)
ds = _require_pyarrow()
dataset = ds.dataset(str(base), format="parquet", partitioning="hive")
names = dataset.schema.names
manifest["total_rows"] = int(dataset.count_rows())
cols = [c for c in ("system_id", "date") if c in names]
if cols:
meta = dataset.to_table(columns=cols).to_pandas()
if "system_id" in meta:
manifest["system_ids"] = sorted(meta["system_id"].dropna().unique().tolist())
if "date" in meta and len(meta):
manifest["min_date"] = str(meta["date"].min())
manifest["max_date"] = str(meta["date"].max())
except Exception: # noqa: BLE001 (summary is best-effort; the hashes are the point)
pass
return manifest
|