Compute

Compute agreement coefficient estimates

compute(df, target, metrics, ci=False, n_boot=1000, confidence_level=0.95, level=LevelType.NOMINAL, models=None, runs=None, **filters)

Compute interrater agreement coefficient estimates and optional confidence intervals.

Wraps data in a Dataset container, applies subset filters if passed, and evaluates the coefficients specified. Non-parametric BCa cluster bootstrapping can be enabled to estimate standard errors and confidence intervals.

Parameters:

Name	Type	Description	Default
df	DataFrame	Long-form dataframe with at minimum, ratings and metadata	required
target	str	Name of the column in df containing ratings	required
metrics	list[str]	List of coefficient identifiers to compute, e.g., ["fleiss", "ac1"]	required
ci	bool	If True, estimates uncertainty with standard errors and BCa confidence intervals via item-level cluster bootstrapping	False
n_boot	Int	Number of bootstrap samples when ci=True	1000
confidence_level	float	Alpha for BCa bootstrap intervals when ci=True, e.g., 0.95 for 95% CIs	0.95
level	str or LevelType	Measurement scale level of the target column	LevelType.NOMINAL
models	list[str]	Optional list of model names to filter the dataset	None
runs	list[str]	Optional list of run IDs to filter the dataset	None
**filters	Any	Additional keyword filters applied to metadata columns in df	{}

Returns:

Type	Description
AgreementResult	Immutable result object containing point estimates, bootstrap standard errors, confidence interval ranges, and dataset metadata

Raises:

Type	Description
ValueError	If any requested metric string is not registered in METRIC_REGISTRY

Examples:

>>> result = compute(
...     df=ratings_df,
...     target="rating",
...     metrics=["cohen", "ac1"],
...     ci=True,
...     models=["gpt-4o", "claude-3-5-sonnet"]
... )
>>> print(result.scores)
{'cohen': 0.742, 'ac1': 0.810}

Source code in interrater/compute.py

def compute(
    df: pd.DataFrame,
    target: str,
    metrics: list[str],
    ci: bool = False,
    n_boot: int = 1000,
    confidence_level: float = 0.95,
    level: str | LevelType = LevelType.NOMINAL,
    models: list[str] | None = None,
    runs: list[str] | None = None,
    **filters: Any
) -> AgreementResult:
    """
    Compute interrater agreement coefficient estimates and optional confidence intervals.

    Wraps data in a Dataset container, applies subset filters if passed, and evaluates
    the coefficients specified. Non-parametric BCa cluster bootstrapping can be enabled
    to estimate standard errors and confidence intervals.

    Parameters
    ----------
    df : pd.DataFrame
        Long-form dataframe with at minimum, ratings and metadata
    target : str
        Name of the column in `df` containing ratings
    metrics : list[str]
        List of coefficient identifiers to compute, e.g., ["fleiss", "ac1"]
    ci:  bool, default=False
        If `True`, estimates uncertainty with standard errors and BCa confidence intervals
        via item-level cluster bootstrapping
    n_boot : Int, default=1000
        Number of bootstrap samples when `ci=True`
    confidence_level : float, default=0.95
        Alpha for BCa bootstrap intervals when `ci=True`, e.g., 0.95 for 95% CIs
    level : str or LevelType, default=LevelType.NOMINAL
        Measurement scale level of the target column
    models : list[str], default=None
        Optional list of model names to filter the dataset
    runs : list[str], default=None
        Optional list of run IDs to filter the dataset
    **filters : Any
        Additional keyword filters applied to metadata columns in `df`

    Returns
    -------
    AgreementResult
        Immutable result object containing point estimates, bootstrap standard errors,
        confidence interval ranges, and dataset metadata

    Raises
    ------
    ValueError
        If any requested metric string is not registered in `METRIC_REGISTRY`

    Examples
    --------
    >>> result = compute(
    ...     df=ratings_df,
    ...     target="rating",
    ...     metrics=["cohen", "ac1"],
    ...     ci=True,
    ...     models=["gpt-4o", "claude-3-5-sonnet"]
    ... )
    >>> print(result.scores)
    {'cohen': 0.742, 'ac1': 0.810}
    """
    # Initialized standardized data container
    dataset = Dataset(df, target=target, level=level)

    # Apply filtering
    slice_filters = {**filters}
    if models is not None:
        slice_filters["model"] = models
    if runs is not None:
        slice_filters["run"] = runs

    if slice_filters:
        dataset = dataset.slice(**slice_filters)

    # Compute and compile scores
    scores = {}
    standard_errors = {}
    ci_ranges = {}
    coefficient_objects = {}

    for metric_name in metrics:
        if metric_name not in METRIC_REGISTRY:
            raise ValueError(
                f"Unknown metric '{metric_name}'. "
                f"Available metrics: {list(METRIC_REGISTRY.keys())}"
            )

        # Instantiate coefficient class
        coefficient = METRIC_REGISTRY[metric_name]()

        # Calculate point estimate
        coefficient_objects[metric_name] = coefficient
        scores[metric_name] = coefficient.compute(dataset)

        # Estimate uncertainty with bootstrap confidence intervals
        if ci:
            bootstrap_result = bootstrap_ci(
                dataset,
                coefficient,
                n_boot=n_boot,
                confidence_level=confidence_level
            )

            standard_errors[metric_name] = bootstrap_result.standard_error

            ci_ranges[metric_name] = (
                bootstrap_result.ci_lower,
                bootstrap_result.ci_upper
            )

    return AgreementResult(
        scores=scores,
        standard_errors=standard_errors,
        ci_ranges=ci_ranges,
        target=dataset.target,
        level=str(dataset.level),
        n_items=len(dataset.items),
        n_raters=len(dataset.df[["model", "run"]].drop_duplicates()),
        models=dataset.models,
        runs=dataset.runs,
        dataset=dataset,
        coefficients= coefficient_objects
    )

compute_many(dataset, fields, metrics, **compute_kwargs)

Compute interrater agreement coefficient estimates and optional confidence intervals across multiple dataset fields/categories simultaneously

Slices a dataset into subsets based on the specified fields/categories and runs compute() on each slice

Parameters:

Name	Type	Description	Default
dataset	Dataset	Dataset instance containing multiple items and categories	required
fields	list[str]	List of categories/fields to slice and evaluate	required
metrics	list[str]	List of coefficient identifiers to compute, e.g., ["fleiss", "ac1"] for each field	required
**compute_kwargs		Keyword arguments forwarded to compute() for each field, e.g., ci=True, n_boot=1000, confidence_level=0.95	{}

Returns:

Type	Description
MultiAgreementResult	Immutable result object containing point estimates, bootstrap standard errors, confidence interval ranges, and dataset metadata for each field

Examples:

>>> multi_res = compute_many(
...     dataset=my_dataset,
...     fields=["recommendations", "hypoglycemia", "follow_up"],
...     metrics=["fleiss", "ac1"],
...     ci=True
... )
>>> multi_res["recommendations"].scores
{'fleiss': 0.62, 'ac1': 0.78}

Source code in interrater/compute.py

def compute_many(
    dataset: Dataset, 
    fields: list[str], 
    metrics: list[str], 
    **compute_kwargs
) -> MultiAgreementResult:
    """
    Compute interrater agreement coefficient estimates and optional confidence intervals
    across multiple dataset fields/categories simultaneously 

    Slices a dataset into subsets based on the specified fields/categories and 
    runs `compute()` on each slice

    Parameters
    ----------
    dataset: Dataset
        `Dataset` instance containing multiple items and categories
    fields: list[str]
        List of categories/fields to slice and evaluate
    metrics: list[str]
        List of coefficient identifiers to compute, e.g., ["fleiss", "ac1"] for each field
    **compute_kwargs: Any
        Keyword arguments forwarded to `compute()` for each field, e.g.,
        `ci=True`, `n_boot=1000`, `confidence_level=0.95`

    Returns
    -------
    MultiAgreementResult
        Immutable result object containing point estimates, bootstrap standard errors,
        confidence interval ranges, and dataset metadata for each field

    Examples
    --------
    >>> multi_res = compute_many(
    ...     dataset=my_dataset,
    ...     fields=["recommendations", "hypoglycemia", "follow_up"],
    ...     metrics=["fleiss", "ac1"],
    ...     ci=True
    ... )
    >>> multi_res["recommendations"].scores
    {'fleiss': 0.62, 'ac1': 0.78}
    """
    results = {}

    for field in fields:
        # Slice the dataset for this category
        sliced_ds = dataset.slice(category=field)

        # Run standard compute
        results[field] = compute(
            df=sliced_ds.df,
            target=dataset.target,
            metrics=metrics,
            **compute_kwargs
        )

    return MultiAgreementResult(results)

compare_to_ground_truth(df, target, gt_source, gt_target, metrics, ci=False, n_boot=1000, confidence_level=0.95, level=LevelType.NOMINAL, **filters)

Compute interrater agreement coefficient estimates between individual models and a ground truth dataset

Combines model outputs with ground truth ratings, isolates each model against the ground truth, and computes the specified metrics

Parameters:

Name	Type	Description	Default
df	PathOrDataFrame	Model outputs provided as a file path, pd.DataFrame, dictionary, or list	required
target	str	Name of the column in df containing ratings	required
gt_source	PathOrDataFrame	Ground truth reference provided as a file path, pd.DataFrame, dictionary, or list	required
gt_target	str	Name of the column in gt_source containing ground truth ratings	required
metrics	list[str]	List of coefficient identifiers to compute, e.g., ["fleiss", "ac1"]	required
ci	bool	If True, estimates uncertainty with standard errors and BCa confidence intervals via item-level cluster bootstrapping	False
n_boot	Int	Number of bootstrap samples when ci=True	1000
confidence_level	float	Alpha for BCa bootstrap intervals when ci=True, e.g., 0.95 for 95% CIs	0.95
level	str | LevelType	Measurement scale level of the target column	NOMINAL
**filters	Any	Additional key-value filters applied to combined evaluation data before comparison	{}

Returns:

Type	Description
dict[str, dict[str, float]]	Nested dictionary structured as {model_name: {metric_name: score}}.

Examples:

>>> gt_comparison = compare_to_ground_truth(
...     df="model_outputs.csv",
...     target="prediction",
...     gt_source="expert_labels.csv",
...     gt_target="consensus_label",
...     metrics=["cohen", "percent_agreement"],
...     ci=True,
...     n_boot=1000,
...     confidence_level=0.95
... )
>>> gt_comparison["gpt-5.4-mini"]
{'cohen': 0.812, 'percent_agreement': 0.890}

Source code in interrater/compute.py

def compare_to_ground_truth(
    df: PathOrDataFrame,
    target: str,
    gt_source: PathOrDataFrame,
    gt_target: str,
    metrics: list[str],
    ci: bool = False,
    n_boot: int = 1000,
    confidence_level: float = 0.95,
    level: str | LevelType = LevelType.NOMINAL,
    **filters: Any
) -> MultiAgreementResult:
    """
    Compute interrater agreement coefficient estimates between individual models
    and a ground truth dataset

    Combines model outputs with ground truth ratings, isolates each model
    against the ground truth, and computes the specified metrics

    Parameters
    ----------
    df: PathOrDataFrame
        Model outputs provided as a file path, pd.DataFrame, dictionary, or list
    target: str
        Name of the column in `df` containing ratings
    gt_source: PathOrDataFrame
        Ground truth reference provided as a file path, pd.DataFrame, dictionary, or list
    gt_target: str
        Name of the column in `gt_source` containing ground truth ratings
    metrics: list[str]
        List of coefficient identifiers to compute, e.g., ["fleiss", "ac1"]
    ci: bool, default=False
        If `True`, estimates uncertainty with standard errors and BCa confidence intervals
        via item-level cluster bootstrapping
    n_boot : Int, default=1000
        Number of bootstrap samples when `ci=True`
    confidence_level : float, default=0.95
        Alpha for BCa bootstrap intervals when `ci=True`, e.g., 0.95 for 95% CIs
    level: str or LevelType, default=LevelType.NOMINAL
        Measurement scale level of the target column
    **filters: Any
        Additional key-value filters applied to combined evaluation data before comparison

    Returns
    -------
    dict[str, dict[str, float]]
        Nested dictionary structured as `{model_name: {metric_name: score}}`.

    Examples
    --------
    >>> gt_comparison = compare_to_ground_truth(
    ...     df="model_outputs.csv",
    ...     target="prediction",
    ...     gt_source="expert_labels.csv",
    ...     gt_target="consensus_label",
    ...     metrics=["cohen", "percent_agreement"],
    ...     ci=True,
    ...     n_boot=1000,
    ...     confidence_level=0.95
    ... )
    >>> gt_comparison["gpt-5.4-mini"]
    {'cohen': 0.812, 'percent_agreement': 0.890}
    """
    # Load data and attach ground truth
    dataset = Dataset.from_source(df, target=target, level=level)
    dataset = dataset.with_ground_truth(gt_source, gt_target)

    # Apply filters if specified
    if filters:
        dataset = dataset.slice(**filters)

    # models = [m for m in dataset.models if m != "ground_truth"]
    eval_df = dataset.df[dataset.df["model"] != "ground_truth"]
    model_runs = (eval_df[["model", "run"]].drop_duplicates().to_records(index=False))

    comparison_results: dict[str, AgreementResult] = {}

    for model_name, run_id in model_runs:
        model_run_df = dataset.df[
            (dataset.df["model"] == "ground_truth") |
            ((dataset.df["model"] == model_name) & (dataset.df["run"] == run_id))
        ]

        field_key = f"{model_name}__run_{run_id}"

        comparison_results[field_key] = compute(
            df=model_run_df,
            target=dataset.target,
            metrics=metrics,
            ci=ci,
            n_boot=n_boot,
            confidence_level=confidence_level,
            level=dataset.level
        )

    return MultiAgreementResult(results=comparison_results)

:::