Leaderboard

A judge's judge() returns a Leaderboard. Declare its schema once with LeaderboardSpec (one MeasureSpec per measure), assemble rows with LeaderboardBuilder, and let build(expected_topic_ids=..., on_missing=...) verify coverage before the table is trusted.

Leaderboard and entries

autojudge_base.leaderboard.Leaderboard dataclass

Leaderboard(measures: Tuple[MeasureName, ...], spec: 'LeaderboardSpec', entries: Tuple[LeaderboardEntry, ...], all_topic_id: str = 'all')

Thin serialization vessel for leaderboard results.

• measures defines the measure names.
• spec contains full MeasureSpecs with dtype and description.
• entries contains per-topic rows and and per-measure all_topic_id rows.

Developer note: - Aggregation logic lives in LeaderboardBuilder.

measures instance-attribute

measures: Tuple[MeasureName, ...]

spec instance-attribute

spec: 'LeaderboardSpec'

entries instance-attribute

entries: Tuple[LeaderboardEntry, ...]

all_topic_id class-attribute instance-attribute

all_topic_id: str = 'all'

all_measure_names

all_measure_names() -> Tuple[MeasureName, ...]

Return measure names in schema order.

Source code in src/autojudge_base/leaderboard/leaderboard.py

def all_measure_names(self) -> Tuple[MeasureName, ...]:
    """Return measure names in schema order."""
    return self.measures

write

write(output: Path, format: LeaderboardFormat = 'ir_measures') -> None

Write the leaderboard as tab-separated lines.

Parameters:

Name	Type	Description	Default
output	Path	Path to write to	required
format	LeaderboardFormat	Column order - "trec_eval": measure topic value - "tot": run measure topic value - "ir_measures": run topic measure value	'ir_measures'

Only measures present in each entry are written (allows sparse rows).

Source code in src/autojudge_base/leaderboard/leaderboard.py

def write(
    self,
    output: Path,
    format: LeaderboardFormat = "ir_measures",
) -> None:
    """
    Write the leaderboard as tab-separated lines.

    Args:
        output: Path to write to
        format: Column order
            - "trec_eval": measure topic value
            - "tot": run measure topic value
            - "ir_measures": run topic measure value

    Only measures present in each entry are written (allows sparse rows).
    """
    lines: List[str] = []
    for e in self.entries:
        for m in self.all_measure_names():
            if m in e.values:
                if format == "tot":
                    lines.append("\t".join([e.run_id, m, e.topic_id, str(e.values[m])]))
                elif format == "trec_eval":
                    lines.append("\t".join([m, e.topic_id, str(e.values[m])]))
                elif format == "ir_measures":
                    lines.append("\t".join([e.run_id, e.topic_id, m, str(e.values[m])]))
                elif format == "rag4reports":
                    lines.append("\t".join([e.topic_id, e.run_id, m, str(e.values[m])]))
                else:
                    raise ValueError(f"Unknown format: {format!r}")

    output.parent.mkdir(parents=True, exist_ok=True)
    print(f"Writing leaderboard to {output.absolute()}")   # ToDo: use a logger
    output.write_text("\n".join(lines) + "\n", encoding="utf-8")

load classmethod

load(path: Path, format: LeaderboardFormat, has_header: bool = False) -> 'Leaderboard'

Load a leaderboard from file or directory.

Parameters:

Name	Type	Description	Default
path	Path	Path to leaderboard file or directory of files	required
format	LeaderboardFormat	Column order (whitespace-separated) - "trec_eval": measure topic value (3 cols, run_id from filename) - "tot": run measure topic value (4 cols) - "ir_measures": run topic measure value (4 cols) - "ranking": topic Q0 doc_id rank score run (6 cols)	required
has_header	bool	If True, skip the first line (header row)	False

If path is a directory, all files are loaded and merged into a single leaderboard. For trec_eval format, each file's name becomes the run_id.

Source code in src/autojudge_base/leaderboard/leaderboard.py

@classmethod
def load(
    cls,
    path: Path,
    format: LeaderboardFormat,
    has_header: bool = False,
) -> "Leaderboard":
    """
    Load a leaderboard from file or directory.

    Args:
        path: Path to leaderboard file or directory of files
        format: Column order (whitespace-separated)
            - "trec_eval": measure topic value (3 cols, run_id from filename)
            - "tot": run measure topic value (4 cols)
            - "ir_measures": run topic measure value (4 cols)
            - "ranking": topic Q0 doc_id rank score run (6 cols)
        has_header: If True, skip the first line (header row)

    If path is a directory, all files are loaded and merged into a single
    leaderboard. For trec_eval format, each file's name becomes the run_id.
    """
    if path.is_dir():
        return cls._load_directory(path, format, has_header)
    else:
        return cls._load_file(path, format, has_header)

verify

verify(on_missing: OnMissing, expected_topic_ids: Sequence[str], warn: Optional[bool] = False)

Source code in src/autojudge_base/leaderboard/leaderboard.py

def verify(self,  on_missing:OnMissing, expected_topic_ids: Sequence[str], warn:Optional[bool]=False):
    LeaderboardVerification(leaderboard = self, warn=warn, expected_topic_ids=expected_topic_ids, on_missing=on_missing) \
        .complete_measures(include_all_row=True) \
        .complete_topics()

autojudge_base.leaderboard.LeaderboardEntry dataclass

LeaderboardEntry(run_id: str, topic_id: str, values: Dict[MeasureName, Any])

One row in a leaderboard: (run_id, topic_id) plus a mapping of measure -> value.

run_id instance-attribute

run_id: str

topic_id instance-attribute

topic_id: str

values instance-attribute

values: Dict[MeasureName, Any]

autojudge_base.leaderboard.LeaderboardFormat module-attribute

LeaderboardFormat = Literal['trec_eval', 'tot', 'ir_measures', 'ranking', 'unknown']

Specs and builder

autojudge_base.leaderboard.MeasureSpec dataclass

MeasureSpec(name: MeasureName, dtype: MeasureDtype = float, description: str = '')

Build-time definition of a measure.

Parameters:

Name	Type	Description	Default
name	MeasureName	Key used in entry.values and output.	required
dtype	MeasureDtype	The data type for this measure. Determines cast, aggregate, and default: - float (default): cast to float, aggregate with mean, default 0.0 - int: cast to int, aggregate with mean (returns float), default 0 - str: cast to str, aggregate with first_value, default ""	float
description	str	Human-readable description of what this measure represents. Exported to measures.yml for documentation.	''

Per-topic values are cast to dtype. Aggregate ("all" row) is always float for numeric types (mean), preserving backwards compatibility.

Examples:

MeasureSpec("SCORE") # float, mean aggregation MeasureSpec("COUNT", int) # int per-topic, float aggregate MeasureSpec("CATEGORY", str) # str, first_value aggregation MeasureSpec("RELEVANCE", description="How relevant the response is to the query")

name instance-attribute

name: MeasureName

dtype class-attribute instance-attribute

dtype: MeasureDtype = float

description class-attribute instance-attribute

description: str = ''

cast property

cast: CastFn

Return cast function based on dtype.

aggregate property

aggregate: AggFn

Return aggregate function based on dtype.

default property

default: Any

Return default value based on dtype.

__post_init__

__post_init__()

Source code in src/autojudge_base/leaderboard/leaderboard.py

def __post_init__(self):
    if self.dtype not in (float, int, str):
        raise ValueError(
            f"MeasureSpec dtype must be float, int, or str, got {self.dtype!r}"
        )

autojudge_base.leaderboard.LeaderboardSpec dataclass

LeaderboardSpec(measures: Tuple[MeasureSpec, ...], all_topic_id: str = 'all')

Build-time schema for a leaderboard.

The spec defines all valid measure names with aggregator and caster. Storing values for different names will raise an error.

measures instance-attribute

measures: Tuple[MeasureSpec, ...]

all_topic_id class-attribute instance-attribute

all_topic_id: str = 'all'

names property

names: Tuple[MeasureName, ...]

Measure names in schema order.

name_set property

name_set: set[MeasureName]

Measure names as a set for fast validation.

cast_values

cast_values(values: Mapping[MeasureName, Any]) -> Dict[MeasureName, Any]

Cast/normalize measure values using each MeasureSpec.cast.

Assumes values contains all required measure keys.

Source code in src/autojudge_base/leaderboard/leaderboard.py

def cast_values(self, values: Mapping[MeasureName, Any]) -> Dict[MeasureName, Any]:
    """
    Cast/normalize measure values using each MeasureSpec.cast.

    Assumes `values` contains all required measure keys.
    """
    return {m.name: m.cast(values[m.name]) for m in self.measures}

to_measures_dict

to_measures_dict() -> List[Dict[str, Any]]

Export measure specs as a list of dicts for YAML serialization.

Returns:

Type	Description
List[Dict[str, Any]]	List of dicts with keys: name, dtype, description (if non-empty)

Source code in src/autojudge_base/leaderboard/leaderboard.py

def to_measures_dict(self) -> List[Dict[str, Any]]:
    """
    Export measure specs as a list of dicts for YAML serialization.

    Returns:
        List of dicts with keys: name, dtype, description (if non-empty)
    """
    result = []
    for m in self.measures:
        entry: Dict[str, Any] = {
            "name": m.name,
            "dtype": m.dtype.__name__,  # "float", "int", or "str"
        }
        if m.description:
            entry["description"] = m.description
        result.append(entry)
    return result

write_measures_yaml

write_measures_yaml(output: Path) -> None

Write measure specs to a YAML file.

Parameters:

Name	Type	Description	Default
output	Path	Path to write measures.yml	required

Source code in src/autojudge_base/leaderboard/leaderboard.py

def write_measures_yaml(self, output: Path) -> None:
    """
    Write measure specs to a YAML file.

    Args:
        output: Path to write measures.yml
    """
    import yaml

    data = {"measures": self.to_measures_dict()}
    output.parent.mkdir(parents=True, exist_ok=True)
    with open(output, "w", encoding="utf-8") as f:
        yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False)

autojudge_base.leaderboard.LeaderboardBuilder

LeaderboardBuilder(spec: LeaderboardSpec)

Builder/assembler for Leaderboard.

Responsibilities: - Collect per-topic rows (hand-filled or record-derived). - Validate measure keys (fail fast on typos/missing keys). - Cast values according to the spec. - Compute synthetic per-run all_topic_id rows using each measure's aggregator.

Create a builder for a specific leaderboard specification.

Source code in src/autojudge_base/leaderboard/leaderboard.py

def __init__(self, spec: LeaderboardSpec):
    """Create a builder for a specific leaderboard specification."""
    self.spec = spec
    self._rows: List[LeaderboardEntry] = []

spec instance-attribute

spec = spec

add

add(*, run_id: str, topic_id: str, values: Optional[Dict[MeasureName, Any]] = None, **kw: Any) -> None

Add one per-topic row.

Provide either: - values={...} (a dict of measure -> value), OR - keyword args (e.g., GRADE=..., IS_MATCH=...).

This method is strict by default: - Unknown measure keys raise KeyError. - Missing measure keys raise KeyError.

Source code in src/autojudge_base/leaderboard/leaderboard.py

def add(
    self,
    *,
    run_id: str,
    topic_id: str,
    values: Optional[Dict[MeasureName, Any]] = None,
    **kw: Any,
) -> None:
    """
    Add one per-topic row.

    Provide either:
    - `values={...}` (a dict of measure -> value), OR
    - keyword args (e.g., GRADE=..., IS_MATCH=...).

    This method is strict by default:
    - Unknown measure keys raise KeyError.
    - Missing measure keys raise KeyError.
    """
    if values is None:
        values = kw
    elif kw:
        raise TypeError("Pass either values= or keyword measures, not both.")

    extra = set(values) - self.spec.name_set
    missing = self.spec.name_set - set(values)
    if extra:
        raise KeyError(f"Unknown measure(s): {sorted(extra)}")
    if missing:
        raise KeyError(f"Missing measure(s): {sorted(missing)}")

    casted = self.spec.cast_values(values)
    self._rows.append(LeaderboardEntry(run_id=run_id, topic_id=topic_id, values=casted))

add_records

add_records(records: Iterable[Any], *, run_id: Callable[[Any], str], topic_id: Callable[[Any], str], get_values: Callable[[Any], Dict[MeasureName, Any]]) -> None

Add multiple rows from an iterable of arbitrary record objects.

The caller supplies functions to extract: - run_id(record) - topic_id(record) - get_values(record) -> {measure_name: value, ...}

Source code in src/autojudge_base/leaderboard/leaderboard.py

def add_records(
    self,
    records: Iterable[Any],
    *,
    run_id: Callable[[Any], str],
    topic_id: Callable[[Any], str],
    get_values: Callable[[Any], Dict[MeasureName, Any]],
) -> None:
    """
    Add multiple rows from an iterable of arbitrary record objects.

    The caller supplies functions to extract:
    - `run_id(record)`
    - `topic_id(record)`
    - `get_values(record)` -> {measure_name: value, ...}
    """
    for r in records:
        self.add(run_id=run_id(r), topic_id=topic_id(r), values=get_values(r))

entries

entries() -> tuple[LeaderboardEntry, ...]

Return the currently staged per-topic entries (no synthetic 'all' rows).

Source code in src/autojudge_base/leaderboard/leaderboard.py

def entries(self) -> tuple[LeaderboardEntry, ...]:
    """Return the currently staged per-topic entries (no synthetic 'all' rows)."""
    return tuple(self._rows)

build

build(expected_topic_ids: Optional[Sequence[str]] = None, on_missing: OnMissing = 'default') -> Leaderboard

Build a Leaderboard with synthetic per-run all_topic_id rows.

The returned Leaderboard contains: - all per-topic rows that were added - plus one additional row per run_id with topic_id == spec.all_topic_id

Parameters:

Name	Type	Description	Default
expected_topic_ids	Optional[Sequence[str]]	If provided, handles missing (run_id, topic_id) pairs.	None
on_missing	OnMissing	When expected_topic_ids is provided and gaps exist: - "default": silently create per-topic entries with defaults - "warn": create per-topic entries with defaults and print warning - "fix_aggregate": only fill defaults for "all" row aggregation (no per-topic entries) - "error": raise ValueError listing missing (run_id, topic_id) pairs	'default'

Source code in src/autojudge_base/leaderboard/leaderboard.py

def build(
    self,
    expected_topic_ids: Optional[Sequence[str]] = None,
    on_missing: OnMissing = "default",
) -> Leaderboard:
    """
    Build a Leaderboard with synthetic per-run `all_topic_id` rows.

    The returned Leaderboard contains:
    - all per-topic rows that were added
    - plus one additional row per run_id with topic_id == spec.all_topic_id

    Args:
        expected_topic_ids: If provided, handles missing (run_id, topic_id) pairs.
        on_missing: When expected_topic_ids is provided and gaps exist:
            - "default": silently create per-topic entries with defaults
            - "warn": create per-topic entries with defaults and print warning
            - "fix_aggregate": only fill defaults for "all" row aggregation (no per-topic entries)
            - "error": raise ValueError listing missing (run_id, topic_id) pairs
    """
    # Step 1: Detect missing pairs
    all_missing: List[tuple[str, str]] = []
    if expected_topic_ids is not None:
        all_missing = self._detect_missing_run_topic(expected_topic_ids)

    # Step 2: Handle missing based on mode
    filled_rows: List[LeaderboardEntry] = []
    phantom_defaults: List[tuple[str, str]] = []

    if all_missing:
        formatted_pairs = [f"({r}, {t})" for r, t in sorted(all_missing)]
        if on_missing == "error":
            raise ValueError(
                f"Missing leaderboard entries for {len(all_missing)} (run_id, topic_id) pair(s): {format_preview(formatted_pairs)}"
            )

        if on_missing == "warn":
            print(f"Leaderboard Warning: {len(all_missing)} missing entries: {format_preview(formatted_pairs)}", file=sys.stderr)

        if on_missing in ("default", "warn"):
            # Create actual per-topic entries
            default_values = {ms.name: ms.default for ms in self.spec.measures if ms.default is not None}
            if default_values:
                for run_id, topic_id in all_missing:
                    filled_rows.append(LeaderboardEntry(run_id=run_id, topic_id=topic_id, values=default_values))
        elif on_missing == "fix_aggregate":
            phantom_defaults = all_missing

    # Step 3: Compute aggregates
    all_entries = self._rows + filled_rows
    all_rows = self._compute_aggregates(all_entries, phantom_defaults)

    return Leaderboard(
        measures=self.spec.names,
        spec=self.spec,
        entries=tuple(all_entries + all_rows),
        all_topic_id=self.spec.all_topic_id,
    )

Verification

autojudge_base.leaderboard.LeaderboardVerification

LeaderboardVerification(leaderboard: Leaderboard, on_missing: OnMissing, expected_topic_ids: Optional[Sequence[str]] = None, warn: Optional[bool] = False)

Fluent verifier for leaderboards.

Chain verification methods to run multiple checks:

LeaderboardVerification(leaderboard, on_missing="fix_aggregate").complete_measures().same_topics_per_run()

Or run all checks:

LeaderboardVerification(leaderboard, on_missing="fix_aggregate", expected_topic_ids=topics).all()

Each method raises LeaderboardVerificationError on failure (fail-fast).

Initialize verifier.

Parameters:

Name	Type	Description	Default
leaderboard	Leaderboard	The leaderboard to verify	required
on_missing	OnMissing	How to handle missing entries	required
expected_topic_ids	Optional[Sequence[str]]	Optional set of expected topic IDs	None
warn	Optional[bool]	If True, print warnings instead of raising exceptions	False

Source code in src/autojudge_base/leaderboard/verification.py

def __init__(
    self,
    leaderboard: "Leaderboard",
    on_missing: "OnMissing",
    expected_topic_ids: Optional[Sequence[str]] = None,
    warn: Optional[bool]=False
):
    """
    Initialize verifier.

    Args:
        leaderboard: The leaderboard to verify
        on_missing: How to handle missing entries
        expected_topic_ids: Optional set of expected topic IDs
        warn: If True, print warnings instead of raising exceptions
    """
    self.leaderboard = leaderboard
    self.expected_topic_ids = expected_topic_ids
    self.warn = warn
    self.on_missing = on_missing

leaderboard instance-attribute

leaderboard = leaderboard

expected_topic_ids instance-attribute

expected_topic_ids = expected_topic_ids

warn instance-attribute

warn = warn

on_missing instance-attribute

on_missing = on_missing

complete_measures

complete_measures(include_all_row: bool = True) -> LeaderboardVerification

Verify that every (run_id, topic_id) entry contains all measures.

Parameters:

Name	Type	Description	Default
include_all_row	bool	If True, also check synthetic all_topic_id rows	True

Raises:

Type	Description
LeaderboardVerificationError	If any entry is missing or has extra measures

Source code in src/autojudge_base/leaderboard/verification.py

def complete_measures(self, include_all_row: bool = True) -> "LeaderboardVerification":
    """
    Verify that every (run_id, topic_id) entry contains all measures.

    Args:
        include_all_row: If True, also check synthetic all_topic_id rows

    Raises:
        LeaderboardVerificationError: If any entry is missing or has extra measures
    """
    required = set(self.leaderboard.measures)
    all_topic_id = self.leaderboard.all_topic_id

    missing_reports: list[str] = []
    for e in self.leaderboard.entries:
        if not include_all_row and e.topic_id == all_topic_id:
            continue
        present = set(e.values.keys())
        missing = required - present
        extra = present - required
        if missing or extra:
            parts = []
            if missing:
                parts.append(f"missing={sorted(missing)}")
            if extra:
                parts.append(f"extra={sorted(extra)}")
            missing_reports.append(f"{e.run_id}/{e.topic_id}: " + ", ".join(parts))

    if missing_reports and self.on_missing != "fix_aggregate":
        self._raise_or_warn(LeaderboardVerificationError(
            "Leaderboard entries do not match the measure schema:\n  " + format_preview(missing_reports, limit=25, separator="\n  ")
        ))

    return self

same_topics_per_run

same_topics_per_run(include_all_row: bool = False) -> LeaderboardVerification

Verify that all runs have the same set of topic_ids.

This catches cases where run A has topics {t1,t2} but run B has {t1,t3}, which makes per-run comparisons misleading.

Parameters:

Name	Type	Description	Default
include_all_row	bool	If True, include synthetic all_topic_id rows	False

Raises:

Type	Description
LeaderboardVerificationError	If runs have different topic sets

Source code in src/autojudge_base/leaderboard/verification.py

def same_topics_per_run(self, include_all_row: bool = False) -> "LeaderboardVerification":
    """
    Verify that all runs have the same set of topic_ids.

    This catches cases where run A has topics {t1,t2} but run B has {t1,t3},
    which makes per-run comparisons misleading.

    Args:
        include_all_row: If True, include synthetic all_topic_id rows

    Raises:
        LeaderboardVerificationError: If runs have different topic sets
    """
    all_topic_id = self.leaderboard.all_topic_id
    by_run: dict[str, Set[str]] = {}

    for e in self.leaderboard.entries:
        if not include_all_row and e.topic_id == all_topic_id:
            continue
        by_run.setdefault(e.run_id, set()).add(e.topic_id)

    if not by_run:
        return self

    runs = sorted(by_run.keys())
    reference_run = runs[0]
    reference_topics = by_run[reference_run]

    diffs: list[str] = []
    for r in runs[1:]:
        tset = by_run[r]
        missing = reference_topics - tset
        extra = tset - reference_topics
        if missing or extra:
            parts = []
            if missing:
                parts.append(f"missing_topics={sorted(missing)}")
            if extra:
                parts.append(f"extra_topics={sorted(extra)}")
            diffs.append(f"{r} vs {reference_run}: " + ", ".join(parts))

    if diffs:
        self._raise_or_warn(LeaderboardVerificationError(
            f"Runs do not share the same topic set (reference={reference_run}):\n  " + format_preview(diffs, limit=25, separator="\n  ")
        ))

    return self

complete_topics

complete_topics(include_all_row: bool = False) -> LeaderboardVerification

Verify that every expected topic has at least one entry.

Requires expected_topic_ids to be set in constructor.

Parameters:

Name	Type	Description	Default
include_all_row	bool	If True, include synthetic all_topic_id rows	False

Raises:

Type	Description
LeaderboardVerificationError	If any expected topic is missing

Source code in src/autojudge_base/leaderboard/verification.py

def complete_topics(self, include_all_row: bool = False) -> "LeaderboardVerification":
    """
    Verify that every expected topic has at least one entry.

    Requires expected_topic_ids to be set in constructor.

    Args:
        include_all_row: If True, include synthetic all_topic_id rows

    Raises:
        LeaderboardVerificationError: If any expected topic is missing
    """
    if self.on_missing == "fix_aggregate":
        # Not checking for complete topics
        return self

    if self.expected_topic_ids is None:
        raise RuntimeError("Must set `expected_topic_ids` to non-null in order to check `complete_topics`")

    all_topic_id = self.leaderboard.all_topic_id
    expected = set(self.expected_topic_ids)


    for run, leaderboard_run_entries in groupby(sorted(self.leaderboard.entries
                                                       , key=lambda e: e.run_id)
                                                , key=lambda e:e.run_id):
        seen = set()

        for e in leaderboard_run_entries:
            if not include_all_row and e.topic_id == all_topic_id:
                continue
            if e.topic_id in expected:
                seen.add(e.topic_id)

        missing = expected - seen
        if missing:
            missing_list = sorted(missing)
            self._raise_or_warn(LeaderboardVerificationError(
                f"Run {run}: Missing leaderboard entries for {len(missing_list)} topic(s): {format_preview(missing_list)}"
            ))

    return self

no_extra_topics

no_extra_topics(include_all_row: bool = False) -> LeaderboardVerification

Verify no entries exist for non-expected topics.

Requires expected_topic_ids to be set in constructor.

Parameters:

Name	Type	Description	Default
include_all_row	bool	If True, include synthetic all_topic_id rows in check	False

Raises:

Type	Description
LeaderboardVerificationError	If entries exist for unknown topics

Source code in src/autojudge_base/leaderboard/verification.py

def no_extra_topics(self, include_all_row: bool = False) -> "LeaderboardVerification":
    """
    Verify no entries exist for non-expected topics.

    Requires expected_topic_ids to be set in constructor.

    Args:
        include_all_row: If True, include synthetic all_topic_id rows in check

    Raises:
        LeaderboardVerificationError: If entries exist for unknown topics
    """
    if self.expected_topic_ids is None:
        return self

    all_topic_id = self.leaderboard.all_topic_id
    expected = set(self.expected_topic_ids)
    extras = set()

    for e in self.leaderboard.entries:
        if not include_all_row and e.topic_id == all_topic_id:
            continue
        if e.topic_id not in expected and e.topic_id != all_topic_id:
            extras.add(e.topic_id)

    if extras:
        extra_list = sorted(extras)
        self._raise_or_warn(LeaderboardVerificationError(
            f"Leaderboard entries for {len(extra_list)} unexpected topic(s): {format_preview(extra_list)}"
        ))

    return self

all

all(include_all_row: bool = True) -> LeaderboardVerification

Run all verification checks.

Checks run in order (fail-fast): 1. complete_measures - every entry has all measures 2. complete_topics - every expected topic has entries (if expected_topic_ids set) 3. no_extra_topics - no entries for unknown topics (if expected_topic_ids set) 4. same_topics_per_run - all runs have same topic set

Parameters:

Name	Type	Description	Default
include_all_row	bool	If True, include synthetic all_topic_id rows in measure check	True

Returns:

Type	Description
LeaderboardVerification	self for chaining

Source code in src/autojudge_base/leaderboard/verification.py

def all(self, include_all_row: bool = True) -> "LeaderboardVerification":
    """
    Run all verification checks.

    Checks run in order (fail-fast):
    1. complete_measures - every entry has all measures
    2. complete_topics - every expected topic has entries (if expected_topic_ids set)
    3. no_extra_topics - no entries for unknown topics (if expected_topic_ids set)
    4. same_topics_per_run - all runs have same topic set

    Args:
        include_all_row: If True, include synthetic all_topic_id rows in measure check

    Returns:
        self for chaining
    """

    if self.expected_topic_ids is not None:
        return (
            self.complete_measures(include_all_row=include_all_row)
            .complete_topics()
            .no_extra_topics()
            # .same_topics_per_run()   # omitted because that is already covered by the previous calls.
        )
    else:
        return (
            self.complete_measures(include_all_row=include_all_row)
            .same_topics_per_run()  # Fall back when we don't know the true topics.
        )

autojudge_base.leaderboard.LeaderboardVerificationError

Bases: Exception

Raised when leaderboard verification fails.