Comparing to ground truth¶

This notebook walks through how to compute coefficient estimates for a rater compared to a ground truth dataset.

Setup¶

In [1]:

# Adding root directory for now before packaging

import sys
from pathlib import Path

sys.path.append(str(Path.cwd().parent.parent))

# Adding root directory for now before packaging import sys from pathlib import Path sys.path.append(str(Path.cwd().parent.parent))

In [2]:

# Imports

import pandas as pd
from interrater.compute import compare_to_ground_truth

# Imports import pandas as pd from interrater.compute import compare_to_ground_truth

Mock data¶

In [3]:

model_output_df = pd.DataFrame([
    # GPT-5.4-mini - Run 1
    {"item": "doc_01", "model": "gpt-5.4-mini", "run": 1, "event": "Hyperglycemia"},
    {"item": "doc_02", "model": "gpt-5.4-mini", "run": 1, "event": "Hypoglycemia"},
    {"item": "doc_03", "model": "gpt-5.4-mini", "run": 1, "event": "Good control"},
    
    # GPT-5.4-mini - Run 2
    {"item": "doc_01", "model": "gpt-5.4-mini", "run": 2, "event": "Hyperglycemia"},
    {"item": "doc_02", "model": "gpt-5.4-mini", "run": 2, "event": "Hyperglycemia"},
    {"item": "doc_03", "model": "gpt-5.4-mini", "run": 2, "event": "Good control"},
    
    # GPT-5.4-nano - Run 1
    {"item": "doc_01", "model": "gpt-5.4-nano", "run": 1, "event": "Hyperglycemia"},
    {"item": "doc_02", "model": "gpt-5.4-nano", "run": 1, "event": "Hypoglycemia"},
    {"item": "doc_03", "model": "gpt-5.4-nano", "run": 1, "event": "Hypoglycemia"},
    
    # GPT-5.4-nano - Run 2
    {"item": "doc_01", "model": "gpt-5.4-nano", "run": 2, "event": "Hyperglycemia"},
    {"item": "doc_02", "model": "gpt-5.4-nano", "run": 2, "event": "Hypoglycemia"},
    {"item": "doc_03", "model": "gpt-5.4-nano", "run": 2, "event": "Good control"},
])

model_output_df = pd.DataFrame([ # GPT-5.4-mini - Run 1 {"item": "doc_01", "model": "gpt-5.4-mini", "run": 1, "event": "Hyperglycemia"}, {"item": "doc_02", "model": "gpt-5.4-mini", "run": 1, "event": "Hypoglycemia"}, {"item": "doc_03", "model": "gpt-5.4-mini", "run": 1, "event": "Good control"}, # GPT-5.4-mini - Run 2 {"item": "doc_01", "model": "gpt-5.4-mini", "run": 2, "event": "Hyperglycemia"}, {"item": "doc_02", "model": "gpt-5.4-mini", "run": 2, "event": "Hyperglycemia"}, {"item": "doc_03", "model": "gpt-5.4-mini", "run": 2, "event": "Good control"}, # GPT-5.4-nano - Run 1 {"item": "doc_01", "model": "gpt-5.4-nano", "run": 1, "event": "Hyperglycemia"}, {"item": "doc_02", "model": "gpt-5.4-nano", "run": 1, "event": "Hypoglycemia"}, {"item": "doc_03", "model": "gpt-5.4-nano", "run": 1, "event": "Hypoglycemia"}, # GPT-5.4-nano - Run 2 {"item": "doc_01", "model": "gpt-5.4-nano", "run": 2, "event": "Hyperglycemia"}, {"item": "doc_02", "model": "gpt-5.4-nano", "run": 2, "event": "Hypoglycemia"}, {"item": "doc_03", "model": "gpt-5.4-nano", "run": 2, "event": "Good control"}, ])

In [4]:

ground_truth_df = pd.DataFrame([
    {"item": "doc_01", "label": "Hyperglycemia"},
    {"item": "doc_02", "label": "Hypoglycemia"},
    {"item": "doc_03", "label": "Good control"},
])

ground_truth_df = pd.DataFrame([ {"item": "doc_01", "label": "Hyperglycemia"}, {"item": "doc_02", "label": "Hypoglycemia"}, {"item": "doc_03", "label": "Good control"}, ])

Run ground truth comparison¶

Note that confidence intervals and standard errors can be bootstrapped using the parameters, ci=True, n_boot, and confidence_level. These are not computed in this example given the very small sample size ($n = 3$ items per model run), which violates the bootstrapping assumption that the sample size is large enough to be representative of the population, usually at minimum $n \geq 30$.

In [5]:

comparison = compare_to_ground_truth(
    df=model_output_df,
    target="event",
    gt_source=ground_truth_df,
    gt_target="label",
    metrics=["cohen", "percent_agreement"],
    level="nominal"
)

comparison

comparison = compare_to_ground_truth( df=model_output_df, target="event", gt_source=ground_truth_df, gt_target="label", metrics=["cohen", "percent_agreement"], level="nominal" ) comparison

Out[5]:

MultiAgreementResult(results={'gpt-5.4-mini__run_1': AgreementResult:
  cohen: 1.0000
  percent_agreement: 1.0000, 'gpt-5.4-mini__run_2': AgreementResult:
  cohen: 0.5000
  percent_agreement: 0.6667, 'gpt-5.4-nano__run_1': AgreementResult:
  cohen: 0.5000
  percent_agreement: 0.6667, 'gpt-5.4-nano__run_2': AgreementResult:
  cohen: 1.0000
  percent_agreement: 1.0000})

In [6]:

# View dataframe for per-run metrics

comparison.to_dataframe()

# View dataframe for per-run metrics comparison.to_dataframe()

Out[6]:

	field	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	gpt-5.4-mini__run_1	cohen	1.000000	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-mini,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}
1	gpt-5.4-mini__run_1	percent_agreement	1.000000	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-mini,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}
2	gpt-5.4-mini__run_2	cohen	0.500000	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-mini,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}
3	gpt-5.4-mini__run_2	percent_agreement	0.666667	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-mini,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}
4	gpt-5.4-nano__run_1	cohen	0.500000	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-nano,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}
5	gpt-5.4-nano__run_1	percent_agreement	0.666667	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-nano,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}
6	gpt-5.4-nano__run_2	cohen	1.000000	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-nano,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}
7	gpt-5.4-nano__run_2	percent_agreement	1.000000	NaN	NaN	NaN	event	LevelType.NOMINAL	3	2	gpt-5.4-nano,ground_truth	Dataset(n=6, target='event', level='LevelType....	{}

In [7]:

# Aggregate metrics across model runs

comparison.aggregate_runs()

# Aggregate metrics across model runs comparison.aggregate_runs()

Out[7]:

	model_name	metric	mean_estimate	std_estimate	min_estimate	max_estimate	n_runs
0	gpt-5.4-mini	cohen	0.7500	0.3536	0.5000	1.0	2
1	gpt-5.4-mini	percent_agreement	0.8333	0.2357	0.6667	1.0	2
2	gpt-5.4-nano	cohen	0.7500	0.3536	0.5000	1.0	2
3	gpt-5.4-nano	percent_agreement	0.8333	0.2357	0.6667	1.0	2