Skip to content

generate_manifest

generate_manifest

generate_manifest(lake_dir: str | Path, *, chunk_size: int = 1 << 20) -> dict

A cryptographic manifest of a Parquet lake, for citable, reproducible datasets.

Walks every *.parquet partition file, records its SHA-256 and size, and summarises the dataset (systems, date span, row count). Drop the returned dict next to a Zenodo / Dataverse deposit so a reviewer can verify byte-for-byte what was analysed.

Returns:

Type Description
dict

gbfs_toolkit_version, generated_at (UTC ISO), n_files, total_bytes, total_rows, system_ids, min_date, max_date, and files (a sorted list of {path, sha256, bytes} with paths relative to lake_dir).

See Also

append_to_parquet : The append step it documents. coverage_report : Per-feed coverage summary.

Source code in src/gbfs_toolkit/io/timeseries.py
def generate_manifest(lake_dir: str | Path, *, chunk_size: int = 1 << 20) -> dict:
    """A cryptographic manifest of a Parquet lake, for citable, reproducible datasets.

    Walks every ``*.parquet`` partition file, records its SHA-256 and size, and summarises
    the dataset (systems, date span, row count). Drop the returned dict next to a Zenodo /
    Dataverse deposit so a reviewer can verify byte-for-byte what was analysed.

    Returns
    -------
    dict
        ``gbfs_toolkit_version``, ``generated_at`` (UTC ISO), ``n_files``, ``total_bytes``,
        ``total_rows``, ``system_ids``, ``min_date``, ``max_date``, and ``files`` (a sorted
        list of ``{path, sha256, bytes}`` with paths relative to ``lake_dir``).

    See Also
    --------
    [`append_to_parquet`][gbfs_toolkit.append_to_parquet] : The append step it documents.
    [`coverage_report`][gbfs_toolkit.coverage_report] : Per-feed coverage summary.
    """
    from gbfs_toolkit import __version__

    base = Path(lake_dir)
    files = []
    for p in sorted(base.rglob("*.parquet")):
        h = hashlib.sha256()
        with p.open("rb") as fh:
            for block in iter(functools.partial(fh.read, chunk_size), b""):
                h.update(block)
        files.append(
            {"path": str(p.relative_to(base)), "sha256": h.hexdigest(), "bytes": p.stat().st_size}
        )

    manifest: dict[str, Any] = {
        "gbfs_toolkit_version": __version__,
        "generated_at": pd.Timestamp.now(tz="UTC").isoformat(),
        "n_files": len(files),
        "total_bytes": sum(f["bytes"] for f in files),
        "files": files,
    }
    try:  # dataset-level summary (best-effort; needs a readable dataset)
        ds = _require_pyarrow()
        dataset = ds.dataset(str(base), format="parquet", partitioning="hive")
        names = dataset.schema.names
        manifest["total_rows"] = int(dataset.count_rows())
        cols = [c for c in ("system_id", "date") if c in names]
        if cols:
            meta = dataset.to_table(columns=cols).to_pandas()
            if "system_id" in meta:
                manifest["system_ids"] = sorted(meta["system_id"].dropna().unique().tolist())
            if "date" in meta and len(meta):
                manifest["min_date"] = str(meta["date"].min())
                manifest["max_date"] = str(meta["date"].max())
    except Exception:  # noqa: BLE001 (summary is best-effort; the hashes are the point)
        pass
    return manifest