Evaluating LLM structured outputs¶

This notebook walks through how to compute metrics and compare raters from structured JSON files, demonstrating how to prepare data and retrieve metadata from filenames, if needed.

Setup¶

In [1]:

%load_ext autoreload
%autoreload 2

%load_ext autoreload %autoreload 2

In [4]:

# Adding root directory for now before packaging

import sys
from pathlib import Path

sys.path.append(str(Path.cwd().parent.parent))

# Adding root directory for now before packaging import sys from pathlib import Path sys.path.append(str(Path.cwd().parent.parent))

In [5]:

# Imports

import json
import pandas as pd
import re

from interrater.base.dataset import Dataset
from interrater.compute import compute, compute_many
from interrater.evaluate import evaluate

# Imports import json import pandas as pd import re from interrater.base.dataset import Dataset from interrater.compute import compute, compute_many from interrater.evaluate import evaluate

In [4]:

# Directories

data_dir = Path("sample_data/json")
json_files = list(data_dir.rglob("*.json"))

# Directories data_dir = Path("sample_data/json") json_files = list(data_dir.rglob("*.json"))

Prepare data¶

In [5]:

# Find files and flatten

filename_regex = re.compile(r"parsed_output_(.+)_([^_]+)_(\d+)\.json")

flattened_rows = []

# Extract all structured output targets, excluding free text fields
for file_path in json_files:
    match = filename_regex.match(file_path.name)
    if not match:
        continue

    model, id, run_idx = match.groups()
    run_idx = int(run_idx)

    try:
        with open(file_path, "r") as f:
            data = json.load(f)

            # Target 1: recommendations
            recs = data.get("recommendations", {})
            for rec_name, judgment in recs.items():
                flattened_rows.append({
                    "item": f"{id}_rec_{rec_name}",
                    "model": model,
                    "run": run_idx,
                    "rating": judgment,
                    "category": "recommendations" # field for slicing
                })

            # Target 2: nocturnal hypoglycemia
            if "nocturnal_hypoglycemia" in data:
                flattened_rows.append({
                    "item": f"{id}_nocturnal_hypoglycemia",
                    "model": model,
                    "run": run_idx,
                    "rating": data["nocturnal_hypoglycemia"],
                    "category": "nocturnal_hypoglycemia" 
                })

            # Target 3: earlier follow up
            if "earlier_follow_up" in data:
                flattened_rows.append({
                    "item": f"{id}_earlier_follow_up",
                    "model": model,
                    "run": run_idx,
                    "rating": data["earlier_follow_up"],
                    "category": "earlier_follow_up" 
                })
            
            # Target 4: recurring daily patterns
            signatures = []
            for obj in data.get("recurring_daily_patterns", []):
                sig = "_".join([str(v) for k, v in obj.items() if k != "rank"])
                signatures.append(sig)
                flattened_rows.append({
                    "item": f"{id}_daily_pattern_{obj.get('rank')}",
                    "model": model,
                    "run": run_idx,
                    "rating": tuple(signatures), # Store as tuple of strings for set coefficients
                    "category": "daily_patterns"
                })

            # Target 5: findings by day
            signatures = []
            for obj in data.get("findings_by_day", []):
                sig = "_".join([str(v) for k, v in obj.items() if k != "rank"])
                signatures.append(sig)
                flattened_rows.append({
                    "item": f"{id}_findings_{obj.get('rank')}",
                    "model": model,
                    "run": run_idx,
                    "rating": tuple(signatures),
                    "category": "findings_by_day"
                })

            # Target 6: top recommendations
            top_recs = data.get("top_recommendations", [])
            if top_recs:
                flattened_rows.append({
                    "item": f"{id}_top_recs",
                    "model": model,
                    "run": run_idx,
                    "rating": tuple(top_recs),  
                    "category": "top_recommendations"
                })

    except Exception:
        continue

# Convert to dataframe
df_fields = pd.DataFrame(flattened_rows)
dataset = Dataset(_df=df_fields, target="rating")
print(dataset)

# Find files and flatten filename_regex = re.compile(r"parsed_output_(.+)_([^_]+)_(\d+)\.json") flattened_rows = [] # Extract all structured output targets, excluding free text fields for file_path in json_files: match = filename_regex.match(file_path.name) if not match: continue model, id, run_idx = match.groups() run_idx = int(run_idx) try: with open(file_path, "r") as f: data = json.load(f) # Target 1: recommendations recs = data.get("recommendations", {}) for rec_name, judgment in recs.items(): flattened_rows.append({ "item": f"{id}_rec_{rec_name}", "model": model, "run": run_idx, "rating": judgment, "category": "recommendations" # field for slicing }) # Target 2: nocturnal hypoglycemia if "nocturnal_hypoglycemia" in data: flattened_rows.append({ "item": f"{id}_nocturnal_hypoglycemia", "model": model, "run": run_idx, "rating": data["nocturnal_hypoglycemia"], "category": "nocturnal_hypoglycemia" }) # Target 3: earlier follow up if "earlier_follow_up" in data: flattened_rows.append({ "item": f"{id}_earlier_follow_up", "model": model, "run": run_idx, "rating": data["earlier_follow_up"], "category": "earlier_follow_up" }) # Target 4: recurring daily patterns signatures = [] for obj in data.get("recurring_daily_patterns", []): sig = "_".join([str(v) for k, v in obj.items() if k != "rank"]) signatures.append(sig) flattened_rows.append({ "item": f"{id}_daily_pattern_{obj.get('rank')}", "model": model, "run": run_idx, "rating": tuple(signatures), # Store as tuple of strings for set coefficients "category": "daily_patterns" }) # Target 5: findings by day signatures = [] for obj in data.get("findings_by_day", []): sig = "_".join([str(v) for k, v in obj.items() if k != "rank"]) signatures.append(sig) flattened_rows.append({ "item": f"{id}_findings_{obj.get('rank')}", "model": model, "run": run_idx, "rating": tuple(signatures), "category": "findings_by_day" }) # Target 6: top recommendations top_recs = data.get("top_recommendations", []) if top_recs: flattened_rows.append({ "item": f"{id}_top_recs", "model": model, "run": run_idx, "rating": tuple(top_recs), "category": "top_recommendations" }) except Exception: continue # Convert to dataframe df_fields = pd.DataFrame(flattened_rows) dataset = Dataset(_df=df_fields, target="rating") print(dataset)

Dataset(n=89, target='rating', level='LevelType.NOMINAL', models=2, items=23)

In [6]:

# View underlying dataframe

dataset.df

# View underlying dataframe dataset.df

Out[6]:

	item	model	run	rating
0	108052_rec_Prevent nocturnal hypoglycemia	gpt-5.4-mini	0	Oppose for relevance
1	108052_rec_Increase insulin to carb ratio	gpt-5.4-mini	0	Support
2	108052_rec_Decrease insulin to carb ratio	gpt-5.4-mini	0	Oppose for safety
3	108052_rec_Improve prandial insulin timing	gpt-5.4-mini	0	Support
4	108052_rec_Improve prandial insulin adherence	gpt-5.4-mini	0	Support
...	...	...	...	...
84	108052_findings_2	gpt-5.4-mini	1	(2026-02-05_Prandial hyperglycemia, 2026-02-06...
85	108052_findings_3	gpt-5.4-mini	1	(2026-02-05_Prandial hyperglycemia, 2026-02-06...
86	108052_findings_4	gpt-5.4-mini	1	(2026-02-05_Prandial hyperglycemia, 2026-02-06...
87	108052_findings_5	gpt-5.4-mini	1	(2026-02-05_Prandial hyperglycemia, 2026-02-06...
88	108052_top_recs	gpt-5.4-mini	1	(Improve prandial insulin timing, Carb countin...

89 rows × 4 columns

In [7]:

# View dataset properties

print(
    f"Dataset raters: {dataset.raters}\n",
    f"Dataset models: {dataset.models}\n",
    f"Dataset items: {dataset.items}"
)

# View dataset properties print( f"Dataset raters: {dataset.raters}\n", f"Dataset models: {dataset.models}\n", f"Dataset items: {dataset.items}" )

Dataset raters: [('gpt-5.4-mini', 0), ('gpt-5.4-nano', 1), ('gpt-5.4-nano', 0), ('gpt-5.4-mini', 1)]
 Dataset models: ['gpt-5.4-mini', 'gpt-5.4-nano']
 Dataset items: <StringArray>
[    '108052_rec_Prevent nocturnal hypoglycemia',
     '108052_rec_Increase insulin to carb ratio',
     '108052_rec_Decrease insulin to carb ratio',
    '108052_rec_Improve prandial insulin timing',
 '108052_rec_Improve prandial insulin adherence',
        '108052_rec_Increase basal insulin dose',
        '108052_rec_Decrease basal insulin dose',
       '108052_rec_Prevent daytime hypoglycemia',
                    '108052_rec_Monitor ketones',
    '108052_rec_Prevent prolonged hyperglycemia',
                      '108052_rec_Carb counting',
            '108052_rec_Confirm sensor accuracy',
                 '108052_nocturnal_hypoglycemia',
                      '108052_earlier_follow_up',
                        '108052_daily_pattern_1',
                        '108052_daily_pattern_2',
                        '108052_daily_pattern_3',
                             '108052_findings_1',
                             '108052_findings_2',
                             '108052_findings_3',
                             '108052_findings_4',
                             '108052_findings_5',
                               '108052_top_recs']
Length: 23, dtype: str

Within-model consistency - 1 field, 1 model¶

In [10]:

# Compute within-model consistency for one field and one model

# Slice down using the internal Dataset filtering
rec_dataset = dataset.slice(category="recommendations")
rec_gpt5_dataset = dataset.slice(category="recommendations", model="gpt-5.4-mini")

# View sliced dataframe
rec_dataset.df

# Compute within-model consistency for one field and one model # Slice down using the internal Dataset filtering rec_dataset = dataset.slice(category="recommendations") rec_gpt5_dataset = dataset.slice(category="recommendations", model="gpt-5.4-mini") # View sliced dataframe rec_dataset.df

Out[10]:

	item	model	run	rating
0	108052_rec_Prevent nocturnal hypoglycemia	gpt-5.4-mini	0	Oppose for relevance
1	108052_rec_Increase insulin to carb ratio	gpt-5.4-mini	0	Support
2	108052_rec_Decrease insulin to carb ratio	gpt-5.4-mini	0	Oppose for safety
3	108052_rec_Improve prandial insulin timing	gpt-5.4-mini	0	Support
4	108052_rec_Improve prandial insulin adherence	gpt-5.4-mini	0	Support
5	108052_rec_Increase basal insulin dose	gpt-5.4-mini	0	Oppose for relevance
6	108052_rec_Decrease basal insulin dose	gpt-5.4-mini	0	Oppose for relevance
7	108052_rec_Prevent daytime hypoglycemia	gpt-5.4-mini	0	Oppose for relevance
8	108052_rec_Monitor ketones	gpt-5.4-mini	0	Support
9	108052_rec_Prevent prolonged hyperglycemia	gpt-5.4-mini	0	Support
10	108052_rec_Carb counting	gpt-5.4-mini	0	Support
11	108052_rec_Confirm sensor accuracy	gpt-5.4-mini	0	Oppose for relevance
12	108052_rec_Prevent nocturnal hypoglycemia	gpt-5.4-nano	1	Oppose for relevance
13	108052_rec_Increase insulin to carb ratio	gpt-5.4-nano	1	Oppose for relevance
14	108052_rec_Decrease insulin to carb ratio	gpt-5.4-nano	1	Oppose for relevance
15	108052_rec_Improve prandial insulin timing	gpt-5.4-nano	1	Support
16	108052_rec_Improve prandial insulin adherence	gpt-5.4-nano	1	Support
17	108052_rec_Increase basal insulin dose	gpt-5.4-nano	1	Oppose for relevance
18	108052_rec_Decrease basal insulin dose	gpt-5.4-nano	1	Oppose for relevance
19	108052_rec_Prevent daytime hypoglycemia	gpt-5.4-nano	1	Support
20	108052_rec_Monitor ketones	gpt-5.4-nano	1	Oppose for relevance
21	108052_rec_Prevent prolonged hyperglycemia	gpt-5.4-nano	1	Support
22	108052_rec_Carb counting	gpt-5.4-nano	1	Support
23	108052_rec_Confirm sensor accuracy	gpt-5.4-nano	1	Oppose for relevance
24	108052_rec_Prevent nocturnal hypoglycemia	gpt-5.4-nano	0	Support
25	108052_rec_Increase insulin to carb ratio	gpt-5.4-nano	0	Oppose for relevance
26	108052_rec_Decrease insulin to carb ratio	gpt-5.4-nano	0	Support
27	108052_rec_Improve prandial insulin timing	gpt-5.4-nano	0	Support
28	108052_rec_Improve prandial insulin adherence	gpt-5.4-nano	0	Support
29	108052_rec_Increase basal insulin dose	gpt-5.4-nano	0	Oppose for relevance
30	108052_rec_Decrease basal insulin dose	gpt-5.4-nano	0	Support
31	108052_rec_Prevent daytime hypoglycemia	gpt-5.4-nano	0	Support
32	108052_rec_Monitor ketones	gpt-5.4-nano	0	Support
33	108052_rec_Prevent prolonged hyperglycemia	gpt-5.4-nano	0	Support
34	108052_rec_Carb counting	gpt-5.4-nano	0	Support
35	108052_rec_Confirm sensor accuracy	gpt-5.4-nano	0	Support
36	108052_rec_Prevent nocturnal hypoglycemia	gpt-5.4-mini	1	Oppose for relevance
37	108052_rec_Increase insulin to carb ratio	gpt-5.4-mini	1	Support
38	108052_rec_Decrease insulin to carb ratio	gpt-5.4-mini	1	Oppose for safety
39	108052_rec_Improve prandial insulin timing	gpt-5.4-mini	1	Support
40	108052_rec_Improve prandial insulin adherence	gpt-5.4-mini	1	Support
41	108052_rec_Increase basal insulin dose	gpt-5.4-mini	1	Support
42	108052_rec_Decrease basal insulin dose	gpt-5.4-mini	1	Oppose for relevance
43	108052_rec_Prevent daytime hypoglycemia	gpt-5.4-mini	1	Oppose for relevance
44	108052_rec_Monitor ketones	gpt-5.4-mini	1	Support
45	108052_rec_Prevent prolonged hyperglycemia	gpt-5.4-mini	1	Support
46	108052_rec_Carb counting	gpt-5.4-mini	1	Support
47	108052_rec_Confirm sensor accuracy	gpt-5.4-mini	1	Support

In [11]:

# Run analysis on the dataframe slice
res_gpt5 = compute(
    df=rec_gpt5_dataset.df, 
    target="rating",
    metrics=["fleiss", "percent_agreement"],
    ci=True,
    n_boot=1000
)

# Print results
res_gpt5.__repr__

# Run analysis on the dataframe slice res_gpt5 = compute( df=rec_gpt5_dataset.df, target="rating", metrics=["fleiss", "percent_agreement"], ci=True, n_boot=1000 ) # Print results res_gpt5.__repr__

Out[11]:

<bound method AgreementResult.__repr__ of AgreementResult:
  fleiss: 0.6923 (SE=0.240, 95% CI=[-0.008, 1.000])
  percent_agreement: 0.8333 (SE=0.111, 95% CI=[0.583, 1.000])>

In [12]:

res_gpt5.metrics

res_gpt5.metrics

Out[12]:

['fleiss', 'percent_agreement']

In [18]:

# View result metrics as a dict

res_gpt5.to_dict()

# View result metrics as a dict res_gpt5.to_dict()

Out[18]:

{'fleiss': np.float64(0.6923076923076924),
 'percent_agreement': 0.8333333333333334}

In [19]:

# View full result objects as dataframe

res_gpt5.to_dataframe()

# View full result objects as dataframe res_gpt5.to_dataframe()

Out[19]:

	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	fleiss	0.692308	0.231591	0.067649	1.0	rating	LevelType.NOMINAL	12	2	gpt-5.4-mini	Dataset(n=24, target='rating', level='LevelTyp...	{}
1	percent_agreement	0.833333	0.103177	0.500000	1.0	rating	LevelType.NOMINAL	12	2	gpt-5.4-mini	Dataset(n=24, target='rating', level='LevelTyp...	{}

Within-model consistency - multiple fields and metrics for 1 model¶

In [8]:

# Slice dataset
gptmini_ds = dataset.slice(model="gpt-5.4-mini")

# Use `compute_many` for multiple fields and metrics
res_gptmini = compute_many(
    dataset=gptmini_ds,
    fields=["recommendations", "nocturnal_hypoglycemia", "earlier_follow_up"],
    metrics=["cohen", "fleiss", "ac1", "scott", "percent_agreement"]
)

res_gptmini.__repr__

# Slice dataset gptmini_ds = dataset.slice(model="gpt-5.4-mini") # Use `compute_many` for multiple fields and metrics res_gptmini = compute_many( dataset=gptmini_ds, fields=["recommendations", "nocturnal_hypoglycemia", "earlier_follow_up"], metrics=["cohen", "fleiss", "ac1", "scott", "percent_agreement"] ) res_gptmini.__repr__

Out[8]:

<bound method MultiAgreementResult.__repr__ of MultiAgreementResult(results={'recommendations': AgreementResult:
  cohen: 0.7000
  fleiss: 0.6923
  ac1: 0.7714
  scott: 0.6923
  percent_agreement: 0.8333, 'nocturnal_hypoglycemia': AgreementResult:
  cohen: 1.0000
  fleiss: 1.0000
  ac1: 1.0000
  scott: 1.0000
  percent_agreement: 1.0000, 'earlier_follow_up': AgreementResult:
  cohen: 1.0000
  fleiss: 1.0000
  ac1: 1.0000
  scott: 1.0000
  percent_agreement: 1.0000})>

In [9]:

# View dataframe

res_gptmini.to_dataframe()

# View dataframe res_gptmini.to_dataframe()

Out[9]:

	field	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	recommendations	cohen	0.700000	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	2	gpt-5.4-mini	Dataset(n=24, target='rating', level='LevelTyp...	{}
1	recommendations	fleiss	0.692308	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	2	gpt-5.4-mini	Dataset(n=24, target='rating', level='LevelTyp...	{}
2	recommendations	ac1	0.771429	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	2	gpt-5.4-mini	Dataset(n=24, target='rating', level='LevelTyp...	{}
3	recommendations	scott	0.692308	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	2	gpt-5.4-mini	Dataset(n=24, target='rating', level='LevelTyp...	{}
4	recommendations	percent_agreement	0.833333	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	2	gpt-5.4-mini	Dataset(n=24, target='rating', level='LevelTyp...	{}
5	nocturnal_hypoglycemia	cohen	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
6	nocturnal_hypoglycemia	fleiss	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
7	nocturnal_hypoglycemia	ac1	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
8	nocturnal_hypoglycemia	scott	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
9	nocturnal_hypoglycemia	percent_agreement	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
10	earlier_follow_up	cohen	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
11	earlier_follow_up	fleiss	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
12	earlier_follow_up	ac1	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
13	earlier_follow_up	scott	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}
14	earlier_follow_up	percent_agreement	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	2	gpt-5.4-mini	Dataset(n=2, target='rating', level='LevelType...	{}

Between-model agreement - 1 field, 2 models¶

In [20]:

# Compute between-model agreement for one field and 2 models

between = compute(
    rec_dataset.df,
    target="rating",
    metrics=["fleiss", "percent_agreement"],
    models=["gpt-5.4-mini", "gpt-5.4-nano"], # if using all models in the dataset, models doesn't need to be specified
    ci=True,
    n_boot=1000
)

# Print results
between.__repr__

# Compute between-model agreement for one field and 2 models between = compute( rec_dataset.df, target="rating", metrics=["fleiss", "percent_agreement"], models=["gpt-5.4-mini", "gpt-5.4-nano"], # if using all models in the dataset, models doesn't need to be specified ci=True, n_boot=1000 ) # Print results between.__repr__

Out[20]:

<bound method AgreementResult.__repr__ of AgreementResult:
  fleiss: 0.2068 (SE=0.110, 95% CI=[0.005, 0.424])
  percent_agreement: 0.5972 (SE=0.087, 95% CI=[0.431, 0.792])>

In [21]:

# View as a dataframe

between.to_dataframe()

# View as a dataframe between.to_dataframe()

Out[21]:

	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	fleiss	0.206838	0.109736	0.004552	0.424000	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}
1	percent_agreement	0.597222	0.087359	0.430556	0.791667	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}

Between-model agreement - multiple fields, multiple models¶

Use the compute_many function to compute agreement metrics for multiple targets. Note that all of the arguments for the standard compute function will also work for the compute_many function, e.g., setting ci=True and n_boot=1000. However, these arguments will throw an error for this data since n_items=1 for the nocturnal hypoglycemia and earlier follow up fields; running bootstrap sampling requires at least 2 samples per item.

In [13]:

nominal_fields = ['recommendations', 'nocturnal_hypoglycemia', 'earlier_follow_up']

nominal_results = compute_many(
    dataset=dataset,
    fields=nominal_fields,
    metrics=["fleiss", "ac1"],
    models=["gpt-5.4-mini", "gpt-5.4-nano"]
)

# Print results
nominal_results

nominal_fields = ['recommendations', 'nocturnal_hypoglycemia', 'earlier_follow_up'] nominal_results = compute_many( dataset=dataset, fields=nominal_fields, metrics=["fleiss", "ac1"], models=["gpt-5.4-mini", "gpt-5.4-nano"] ) # Print results nominal_results

Out[13]:

MultiAgreementResult(results={'recommendations': AgreementResult:
  fleiss: 0.2068
  ac1: 0.4602, 'nocturnal_hypoglycemia': AgreementResult:
  fleiss: -0.3333
  ac1: 0.2000, 'earlier_follow_up': AgreementResult:
  fleiss: 1.0000
  ac1: 1.0000})

In [14]:

# View dataframe

nominal_results.to_dataframe()

# View dataframe nominal_results.to_dataframe()

Out[14]:

	field	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	recommendations	fleiss	0.206838	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}
1	recommendations	ac1	0.460151	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}
2	nocturnal_hypoglycemia	fleiss	-0.333333	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
3	nocturnal_hypoglycemia	ac1	0.200000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
4	earlier_follow_up	fleiss	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
5	earlier_follow_up	ac1	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}

If variance is 0 across all runs, Fleiss's kappa produces NaN, as per the statsmodels implementation we're using.

In [15]:

set_fields = ["top_recommendations", "daily_patterns", "findings_by_day"]

set_results = compute_many(
    dataset=dataset,
    fields=set_fields,
    metrics=['jaccard', 'masi'],
    models=["gpt-5.4-mini", "gpt-5.4-nano"]
)

# Print results
set_results

set_fields = ["top_recommendations", "daily_patterns", "findings_by_day"] set_results = compute_many( dataset=dataset, fields=set_fields, metrics=['jaccard', 'masi'], models=["gpt-5.4-mini", "gpt-5.4-nano"] ) # Print results set_results

Out[15]:

MultiAgreementResult(results={'top_recommendations': AgreementResult:
  jaccard: 0.0000
  masi: 0.0000, 'daily_patterns': AgreementResult:
  jaccard: 0.1111
  masi: 0.1111, 'findings_by_day': AgreementResult:
  jaccard: 0.0667
  masi: 0.0667})

In [16]:

# View dataframe

set_results.to_dataframe()

# View dataframe set_results.to_dataframe()

Out[16]:

	field	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	top_recommendations	jaccard	0.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
1	top_recommendations	masi	0.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
2	daily_patterns	jaccard	0.111111	NaN	NaN	NaN	rating	LevelType.NOMINAL	3	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=12, target='rating', level='LevelTyp...	{}
3	daily_patterns	masi	0.111111	NaN	NaN	NaN	rating	LevelType.NOMINAL	3	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=12, target='rating', level='LevelTyp...	{}
4	findings_by_day	jaccard	0.066667	NaN	NaN	NaN	rating	LevelType.NOMINAL	5	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=17, target='rating', level='LevelTyp...	{}
5	findings_by_day	masi	0.066667	NaN	NaN	NaN	rating	LevelType.NOMINAL	5	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=17, target='rating', level='LevelTyp...	{}

To get all scores for all fields, we can concatenate the two results using extend, which creates a new MultiAgreementResult object.

In [17]:

combined = nominal_results.extend(set_results)
combined

combined = nominal_results.extend(set_results) combined

Out[17]:

MultiAgreementResult(results={'recommendations': AgreementResult:
  fleiss: 0.2068
  ac1: 0.4602, 'nocturnal_hypoglycemia': AgreementResult:
  fleiss: -0.3333
  ac1: 0.2000, 'earlier_follow_up': AgreementResult:
  fleiss: 1.0000
  ac1: 1.0000, 'top_recommendations': AgreementResult:
  jaccard: 0.0000
  masi: 0.0000, 'daily_patterns': AgreementResult:
  jaccard: 0.1111
  masi: 0.1111, 'findings_by_day': AgreementResult:
  jaccard: 0.0667
  masi: 0.0667})

In [18]:

# View dataframe

combined.to_dataframe()

# View dataframe combined.to_dataframe()

Out[18]:

	field	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	recommendations	fleiss	0.206838	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}
1	recommendations	ac1	0.460151	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}
2	nocturnal_hypoglycemia	fleiss	-0.333333	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
3	nocturnal_hypoglycemia	ac1	0.200000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
4	earlier_follow_up	fleiss	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
5	earlier_follow_up	ac1	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
6	top_recommendations	jaccard	0.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
7	top_recommendations	masi	0.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
8	daily_patterns	jaccard	0.111111	NaN	NaN	NaN	rating	LevelType.NOMINAL	3	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=12, target='rating', level='LevelTyp...	{}
9	daily_patterns	masi	0.111111	NaN	NaN	NaN	rating	LevelType.NOMINAL	3	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=12, target='rating', level='LevelTyp...	{}
10	findings_by_day	jaccard	0.066667	NaN	NaN	NaN	rating	LevelType.NOMINAL	5	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=17, target='rating', level='LevelTyp...	{}
11	findings_by_day	masi	0.066667	NaN	NaN	NaN	rating	LevelType.NOMINAL	5	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=17, target='rating', level='LevelTyp...	{}

To select only certain metrics, we can use the select_metrics function.

In [19]:

selected = combined.select_metrics(metrics=['fleiss', 'ac1'])
selected

selected = combined.select_metrics(metrics=['fleiss', 'ac1']) selected

Out[19]:

MultiAgreementResult(results={'recommendations': AgreementResult:
  fleiss: 0.2068
  ac1: 0.4602, 'nocturnal_hypoglycemia': AgreementResult:
  fleiss: -0.3333
  ac1: 0.2000, 'earlier_follow_up': AgreementResult:
  fleiss: 1.0000
  ac1: 1.0000})

In [20]:

selected.fields

selected.fields

Out[20]:

['recommendations', 'nocturnal_hypoglycemia', 'earlier_follow_up']

In [24]:

# View dataframe

selected.to_dataframe()

# View dataframe selected.to_dataframe()

Out[24]:

	field	metric	score	standard_error	ci_lower	ci_upper	target	level	n_items	n_raters	models	dataset	metadata
0	recommendations	fleiss	0.206838	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}
1	recommendations	ac1	0.460151	NaN	NaN	NaN	rating	LevelType.NOMINAL	12	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=48, target='rating', level='LevelTyp...	{}
2	nocturnal_hypoglycemia	fleiss	-0.333333	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
3	nocturnal_hypoglycemia	ac1	0.200000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
4	earlier_follow_up	fleiss	NaN	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}
5	earlier_follow_up	ac1	1.000000	NaN	NaN	NaN	rating	LevelType.NOMINAL	1	4	gpt-5.4-mini,gpt-5.4-nano	Dataset(n=4, target='rating', level='LevelType...	{}

Compare models with a permutation test - 1 field, 2 models¶

Test A: difference in agreement

The example below tests model A vs model B internal consistency

$H_0 : \theta_A = \theta_B$
$H_a : \theta_A \neq \theta_B$

where $\theta$ is any agreement coefficient

Other $H_a$ examples:

• model a against ground truth vs model b against ground truth

In [10]:

# Compute metrics for raters
res_mini = compute(rec_dataset.slice(model="gpt-5.4-nano").df, target="rating", metrics=["cohen"])
res_nano = compute(rec_dataset.slice(model="gpt-5.4-mini").df, target="rating", metrics=["cohen"])

# Compare rater metrics
comparison = evaluate(
    res_mini, 
    res_nano, 
    metrics=["cohen"], # since we have exactly 2 raters
    n_permutations=1000, 
    alternative="two-sided",
    method="auto" # chooses either exact sampling or monte carlo sampling based on the size of the dataset
)

comparison

# Compute metrics for raters res_mini = compute(rec_dataset.slice(model="gpt-5.4-nano").df, target="rating", metrics=["cohen"]) res_nano = compute(rec_dataset.slice(model="gpt-5.4-mini").df, target="rating", metrics=["cohen"]) # Compare rater metrics comparison = evaluate( res_mini, res_nano, metrics=["cohen"], # since we have exactly 2 raters n_permutations=1000, alternative="two-sided", method="auto" # chooses either exact sampling or monte carlo sampling based on the size of the dataset ) comparison

Out[10]:

EvaluationResults(results=[EvaluationResult(metric='cohen', estimate=-0.45000000000000007, p_value=np.float64(0.28142543324383695), null_distribution=array([-0.45, -0.45, -0.45, ...,  0.45,  0.45,  0.45], shape=(4096,)), alternative='two-sided', method='exact', label_a='gpt-5.4-nano', label_b='gpt-5.4-mini')])

In [11]:

# View dataframe

comparison.to_dataframe()

# View dataframe comparison.to_dataframe()

Out[11]:

	raters	metric	estimate	p-value	significant	null_mean	null_std	alternative	method
0	gpt-5.4-nano, gpt-5.4-mini	cohen	-0.45	0.281425	False	0.0	0.353412	two-sided	exact

In [12]:

# View significant results at specified alpha

comparison.significant(alpha=0.01)

# View significant results at specified alpha comparison.significant(alpha=0.01)

Out[12]:

EvaluationResults(results=[])

In [13]:

# Plot the permutation distribution of the test statistic

comparison.plot_distribution("cohen")

# Plot the permutation distribution of the test statistic comparison.plot_distribution("cohen")

Out[13]:

<Axes: title={'center': 'cohen: gpt-5.4-nano, gpt-5.4-mini'}, xlabel='Permutation test statistic', ylabel='Frequency'>

In [14]:

# Inspect the null distribution to ensure the p-value is appropriate

comparison.inspect("cohen")

# Inspect the null distribution to ensure the p-value is appropriate comparison.inspect("cohen")

Out[14]:

	statistic	value
0	-0.450000	observed
1	-0.855422	null_min
2	0.855422	null_max
3	0.000000	null_mean
4	0.353412	null_std
5	224.000000	near_observed_count

In [16]:

comparison.to_dataframe()

comparison.to_dataframe()

Out[16]:

	raters	metric	estimate	p-value	significant	null_mean	null_std	alternative	method
0	gpt-5.4-nano, gpt-5.4-mini	cohen	-0.45	0.281425	False	0.0	0.353412	two-sided	exact

Test B: one-sided test

The example below tests whether internal model consistency is statistically significant.

$H_0 : \theta = 0$, where $\theta$ is any agreement coefficient
$H_a : \theta > 0$

Other $H_a$ examples

• Model vs ground truth > 0
• Between-model agreement > 0

In [13]:

res_mini = compute(rec_dataset.slice(model="gpt-5.4-nano").df, target="rating", metrics=["cohen"])

consistency = evaluate(
    result_a=res_mini,
    result_b=None,
    metrics=["cohen"],
    n_permutations=1000,
    alternative="greater",
    method="auto"
)

res_mini = compute(rec_dataset.slice(model="gpt-5.4-nano").df, target="rating", metrics=["cohen"]) consistency = evaluate( result_a=res_mini, result_b=None, metrics=["cohen"], n_permutations=1000, alternative="greater", method="auto" )

In [14]:

consistency.to_dataframe()

consistency.to_dataframe()

Out[14]:

	raters	metric	estimate	p-value	significant	null_mean	null_std	alternative	method
0	gpt-5.4-nano vs None	cohen	0.25	0.090909	False	0.141238	0.035906	greater	exact

In [17]:

consistency.inspect('cohen')

consistency.inspect('cohen')

Out[17]:

	statistic	value
0	0.250000	observed
1	0.117647	null_min
2	0.250000	null_max
3	0.141238	null_mean
4	0.035906	null_std
5	2.000000	near_observed_count