Skip to content

cohen_kappa

cohen_kappa

cohen_kappa(a, b) -> float

Cohen's kappa between two raters over paired nominal labels.

Pairs with a missing label in either rater are dropped.

See Also

krippendorff_alpha : Multi-rater chance-corrected agreement. wilson_interval : A proportion confidence interval.

Examples:

>>> cohen_kappa([1, 2, 3, 1], [1, 2, 3, 1])  # perfect agreement
1.0
Source code in src/gbfs_toolkit/analytics/agreement.py
def cohen_kappa(a, b) -> float:
    """Cohen's kappa between two raters over paired nominal labels.

    Pairs with a missing label in either rater are dropped.

    See Also
    --------
    [`krippendorff_alpha`][gbfs_toolkit.krippendorff_alpha] : Multi-rater chance-corrected agreement.
    [`wilson_interval`][gbfs_toolkit.wilson_interval] : A proportion confidence interval.

    Examples
    --------
    >>> cohen_kappa([1, 2, 3, 1], [1, 2, 3, 1])  # perfect agreement
    1.0
    """
    sa = pd.Series(list(a)).reset_index(drop=True)
    sb = pd.Series(list(b)).reset_index(drop=True)
    keep = sa.notna() & sb.notna()
    sa, sb = sa[keep], sb[keep]
    n = len(sa)
    if n == 0:
        return float("nan")
    po = float((sa.to_numpy() == sb.to_numpy()).mean())
    cats = pd.Index(sorted(set(sa) | set(sb)))
    pa = sa.value_counts(normalize=True).reindex(cats).fillna(0.0)
    pb = sb.value_counts(normalize=True).reindex(cats).fillna(0.0)
    pe = float((pa * pb).sum())
    if pe >= 1.0:
        return 1.0
    return (po - pe) / (1 - pe)