Skip to content

Data models

The objects a judge reads: one Report per system response, one Request per topic, and Document entries for cited or retrieved text. Because tracks differ in how they attach citations, the sentence types below normalize through Report.get_sentences_with_citations().

Report

autojudge_base.report.Report pydantic-model

Bases: BaseModel

Fields:

is_ragtime pydantic-field 

is_ragtime: bool = True

metadata pydantic-field 

metadata: ReportMetaData

evaldata pydantic-field 

evaldata: Optional[Dict[str, Any]] = None

responses pydantic-field 

responses: Optional[List[NeuclirReportSentence] | List[RagtimeReportSentence] | List[Rag24ReportSentence]] = None

answer pydantic-field 

answer: Optional[List[NeuclirReportSentence] | List[RagtimeReportSentence] | List[Rag24ReportSentence]] = None

path pydantic-field 

path: Optional[Path] = None

references pydantic-field 

references: Optional[List[str]] = None

ranking pydantic-field 

ranking: Optional[List[RetrievedDocuments] | List[RankedDocument]] = None

documents pydantic-field 

documents: Optional[Dict[str, Document]] = None

model_post_init 

model_post_init(__context__: dict | None = None) -> None
Source code in src/autojudge_base/report.py 
140
141
142
143
144
145
146
147
148
149
150
151
def model_post_init(self, __context__: dict | None = None) -> None:
    if self.responses is None:
        # RAG
        self.responses = self.answer

    # RAGTIME validation
    if self.responses is None:
        raise RuntimeError(f"Report does not contain responses or answer: {self}")

    # Expose as RAG format
    if self.answer is None:
        self.answer = self.responses

get_report_text 

get_report_text()
Source code in src/autojudge_base/report.py 
154
155
def get_report_text(self):
    return " ".join([sent.text for sent in self.responses])

get_text 

get_text() -> str
Source code in src/autojudge_base/report.py 
157
158
def get_text(self) -> str:
    return self.get_report_text()

get_paragraphs 

get_paragraphs() -> List[str]

Split report text into paragraphs on double-newlines.

Source code in src/autojudge_base/report.py 
160
161
162
163
164
165
def get_paragraphs(self) -> List[str]:
    """Split report text into paragraphs on double-newlines."""
    import re
    text = self.get_text()
    normalized = re.sub(r'\n\s*\n+', '\n\n', text)
    return [p.strip() for p in normalized.split('\n\n') if p.strip()]

get_sentences 

get_sentences() -> List[str]
Source code in src/autojudge_base/report.py 
167
168
def get_sentences(self) ->List[str]:
    return [s.text for s in self.responses]

get_sentences_with_citations 

get_sentences_with_citations() -> List[NeuclirReportSentence]

Get all sentences with citations in unified format.

Returns NeuclirReportSentence objects where citations is List[str] ordered by priority. Does not modify the underlying report data.

Handles all sentence formats: - NeuclirReportSentence: returned as-is - RagtimeReportSentence: citations sorted by confidence (descending) - Rag24ReportSentence: indices resolved to doc_ids via report.references

Source code in src/autojudge_base/report.py 
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
def get_sentences_with_citations(self) -> List[NeuclirReportSentence]:
    """Get all sentences with citations in unified format.

    Returns NeuclirReportSentence objects where citations is List[str]
    ordered by priority. Does not modify the underlying report data.

    Handles all sentence formats:
    - NeuclirReportSentence: returned as-is
    - RagtimeReportSentence: citations sorted by confidence (descending)
    - Rag24ReportSentence: indices resolved to doc_ids via report.references
    """
    references = self.references or []
    result: List[NeuclirReportSentence] = []

    for r in self.responses:
        if isinstance(r, NeuclirReportSentence):
            result.append(r)
        elif isinstance(r, RagtimeReportSentence):
            citation_confidences = r.citations.items() if r.citations else []
            sorted_ids = [eid for eid, conf in sorted(citation_confidences, key=lambda kv: kv[1], reverse=True)]
            result.append(NeuclirReportSentence(text=r.text, citations=sorted_ids, metadata=r.metadata, evaldata=r.evaldata))
        elif isinstance(r, Rag24ReportSentence):
            doc_ids = [references[i] for i in (r.citations or []) if 0 <= i < len(references)]
            result.append(NeuclirReportSentence(text=r.text, citations=doc_ids, metadata=r.metadata, evaldata=r.evaldata))

    return result

autofill_references 

autofill_references()
Source code in src/autojudge_base/report.py 
197
198
199
200
201
202
203
204
205
206
def autofill_references(self):
    ragtime_citation_set:Set[str] = {c for r in self.responses \
                        for c in r.citations.keys()   
                        if isinstance(r, RagtimeReportSentence) \
                    }
    neuclir_citation_set:Set[str] = {c for r in self.responses \
                        for c in r.citations   
                        if isinstance(r, NeuclirReportSentence) \
                    }
    self.references = list(ragtime_citation_set.union(neuclir_citation_set))

switch_responses_to_answer 

switch_responses_to_answer()
Source code in src/autojudge_base/report.py 
208
209
210
def switch_responses_to_answer(self):
    self.answer=self.responses
    self.responses=None

switch_to_neuclir_responses 

switch_to_neuclir_responses()

Convert all sentence formats to NeuclirReportSentence.

After calling this method, all sentences in report.responses will be NeuclirReportSentence with citations as List[str] ordered by priority.

Handles: - RagtimeReportSentence: sorts citations by confidence (descending) - Rag24ReportSentence: resolves indices to doc_ids via report.references - NeuclirReportSentence: passes through unchanged

Source code in src/autojudge_base/report.py 
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
def switch_to_neuclir_responses(self):
    """Convert all sentence formats to NeuclirReportSentence.

    After calling this method, all sentences in report.responses will be
    NeuclirReportSentence with citations as List[str] ordered by priority.

    Handles:
    - RagtimeReportSentence: sorts citations by confidence (descending)
    - Rag24ReportSentence: resolves indices to doc_ids via report.references
    - NeuclirReportSentence: passes through unchanged
    """
    references = self.references or []

    def convert_sentence(r: ReportSentence) -> NeuclirReportSentence:
        if isinstance(r, NeuclirReportSentence):
            return r
        elif isinstance(r, RagtimeReportSentence):
            # Sort by confidence descending
            citation_confidences = r.citations.items() if r.citations else []
            sorted_ids = [eid for eid, conf in sorted(citation_confidences, key=lambda kv: kv[1], reverse=True)]
            return NeuclirReportSentence(text=r.text, citations=sorted_ids, metadata=r.metadata, evaldata=r.evaldata)
        elif isinstance(r, Rag24ReportSentence):
            # Resolve indices to doc_ids
            doc_ids = [references[i] for i in (r.citations or []) if 0 <= i < len(references)]
            return NeuclirReportSentence(text=r.text, citations=doc_ids, metadata=r.metadata, evaldata=r.evaldata)
        else:
            raise RuntimeError(f"Unknown sentence type: {type(r)}")

    def convert_response_sentences(responses: List[ReportSentence]) -> List[NeuclirReportSentence]:
        return [convert_sentence(r) for r in responses]

    if self.responses is not None:
        self.responses = convert_response_sentences(self.responses)
    elif self.answer is not None:
        self.answer = convert_response_sentences(self.answer)
    else:
        raise RuntimeError(f"Either responses or answer must be set to a non-None value, but received: {self}")

verify_ragtime 

verify_ragtime(use_answer: bool = False, check_doc_ids: bool = True, spec=None)
Source code in src/autojudge_base/report.py 
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
def verify_ragtime(self, use_answer:bool = False, check_doc_ids:bool = True, spec=None):
    if spec is not None:
        from autojudge_base.track_spec import verify as _verify_spec
        return _verify_spec(self, spec, use_answer=use_answer)

    def verify_citation_reference():
        citation_set:Set[str] = {c for r in self.responses \
                                    for c in r.citations.keys()   
                                    if isinstance(r, RagtimeReportSentence) \
                                }
        reference_set:Set[str] = set(self.references or [])
        if not citation_set == reference_set:
            raise RuntimeError(f"Ragtime Report format invalid: citations and reference set must match. However, citation set is {citation_set} while reference set is {reference_set}.")

    def verify_citation_confidence_range():
        for r in self.responses:
            for c,v in r.citations.items():
                if v<0.0 or v>100.0:
                    print(f"Warning: Ragtime Report format invalid confidence: citation confidences must be between 0.0 and 100.0, but found confidence {v} in sentence {r}. Limiting to valid range.")
                if v<0.0:
                    r.citations[c]=0.0
                if v>100.0:
                    r.citations[c]=100.0

    def verify_citation_given():
        for r in self.responses:
            if r.citations is None or len(r.citations)==0:
                print(f"WARNING: Ragtime Report format contains empty citations: {r}")

    def verify_citation_doc_id():
        if check_doc_ids:
            pattern = re.compile(r'^[A-Za-z0-9]+(?:-[A-Za-z0-9]+){4}_[A-Za-z0-9]+$')
            for r in self.responses:
                for c,v in r.citations.items():
                    if not bool(pattern.match(c)):
                        print(f"WARNING: Ragtime Report format invalid docid? Citation contains document_id does not match format, maybe this document is from the wrong collection? document_id: {c}, but should look like this: \"47601789-65d8-4706-9bde-fc89fccfdf14_159897\"")


    def verify_task():
        if self.metadata.task is None or not (self.metadata.task == TaskType.ENGLISH or self.metadata.task == TaskType.MULTILINGUAL):
            raise RuntimeError(f"Ragtime Report requires `metadata.task` to be set to either {TaskType.MULTILINGUAL} or {TaskType.ENGLISH}, but is set to {self.metadata.task}")


    verify_task()
    verify_citation_reference()
    verify_citation_confidence_range()
    if self.is_ragtime:
        verify_citation_doc_id()
    verify_citation_given()

    return True

verify 

verify(spec=None, *, request=None, use_answer: bool = False) -> bool

Verify this report against a TrackSpec (structural-only if spec is None).

Delegates to autojudge_base.track_spec.verify. request is needed only for tracks whose length limit is stored per-request (RAGTIME).

Source code in src/autojudge_base/report.py 
304
305
306
307
308
309
310
311
def verify(self, spec=None, *, request=None, use_answer: bool = False) -> bool:
    """Verify this report against a TrackSpec (structural-only if spec is None).

    Delegates to autojudge_base.track_spec.verify. `request` is needed only for
    tracks whose length limit is stored per-request (RAGTIME).
    """
    from autojudge_base.track_spec import verify as _verify_spec
    return _verify_spec(self, spec, request=request, use_answer=use_answer)

verify_rag 

verify_rag(use_answer: bool = False, spec=None) -> bool

Verify against a RAG spec (default: the latest RAG track, rag26).

Backwards-compatible convenience wrapper over verify(spec).

Source code in src/autojudge_base/report.py 
313
314
315
316
317
318
319
def verify_rag(self, use_answer: bool = False, spec=None) -> bool:
    """Verify against a RAG spec (default: the latest RAG track, rag26).

    Backwards-compatible convenience wrapper over `verify(spec)`.
    """
    from autojudge_base.track_spec import verify as _verify_spec, SPECS
    return _verify_spec(self, spec or SPECS["rag26"], use_answer=use_answer)

to_rag 

to_rag(spec=None) -> Report

Convert to a RAG report (default: RAG 2025 generation). See track_spec.to_rag.

Source code in src/autojudge_base/report.py 
321
322
323
324
def to_rag(self, spec=None) -> "Report":
    """Convert to a RAG report (default: RAG 2025 generation). See track_spec.to_rag."""
    from autojudge_base.track_spec import to_rag as _to_rag
    return _to_rag(self, spec)

to_ragtime 

to_ragtime(spec=None) -> Report

Convert to a RAGTIME report (default: RAGTIME 2025 repgen). See track_spec.to_ragtime.

Source code in src/autojudge_base/report.py 
326
327
328
329
def to_ragtime(self, spec=None) -> "Report":
    """Convert to a RAGTIME report (default: RAGTIME 2025 repgen). See track_spec.to_ragtime."""
    from autojudge_base.track_spec import to_ragtime as _to_ragtime
    return _to_ragtime(self, spec)

Report metadata and sentences

autojudge_base.report.ReportMetaData pydantic-model

Bases: BaseModel

Report meta data for requested reports

Config:

  • populate_by_name: True

Fields:

team_id pydantic-field 

team_id: str

run_id pydantic-field 

run_id: str

topic_id pydantic-field 

topic_id: str = None

collection_ids pydantic-field 

collection_ids: Optional[List[str]] = None

task pydantic-field 

task: Optional[TaskType] = None

description pydantic-field 

description: Optional[str] = None

creator pydantic-field 

creator: Dict[str, Any] = None

extra pydantic-field 

extra: Dict[str, Any] = None

use_starter_kit pydantic-field 

use_starter_kit: Optional[int] = None

type pydantic-field 

type: Optional[str] = None

request_id pydantic-field 

request_id: Optional[str] = None

limit pydantic-field 

limit: Optional[int] = None

narrative_id pydantic-field 

narrative_id: Optional[str | int] = None

narrative pydantic-field 

narrative: Optional[str] = None

run_desc pydantic-field 

run_desc: Optional[str] = None

evaldata pydantic-field 

evaldata: Optional[Dict[str, Any]] = None

model_config class-attribute instance-attribute 

model_config = ConfigDict(populate_by_name=True)

set_topic_ids 

set_topic_ids()
Source code in src/autojudge_base/report.py 
52
53
54
def set_topic_ids(self):
    self.narrative_id = self.topic_id
    self.request_id = self.topic_id

set_narrative_text 

set_narrative_text(narratives: Dict[str, Any])
Source code in src/autojudge_base/report.py 
56
57
def set_narrative_text(self,narratives:Dict[str,Any]):
    self.narrative = narratives[self.narrative_id]

set_msmarco_collection_id 

set_msmarco_collection_id()
Source code in src/autojudge_base/report.py 
59
60
def set_msmarco_collection_id(self):
    self.collection_ids = ["msmarco_v2.1_doc_segmented"]

model_post_init 

model_post_init(__context__: dict | None = None) -> None
Source code in src/autojudge_base/report.py 
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def model_post_init(self, __context__: dict | None = None) -> None:
    if self.topic_id is not None and self.narrative_id is not None and str(self.topic_id) != str(self.narrative_id):
        raise ValueError(
            f"Inconsistent topic identifiers: "
            f"topic_id={self.topic_id}, narrative_id={self.narrative_id}"
        )            

    if self.topic_id is None:
        # print("metadata topic_id is None, looking at other fields", self)
        # RAG input
        if self.narrative_id is not None and self.topic_id is None:
            if isinstance(self.narrative_id,int):
                self.topic_id = f"{self.narrative_id}"
            else:
                self.topic_id = self.narrative_id

    if self.topic_id is None:
        raise RuntimeError(f"ReportMetaData does not contain topic_id or narrative_id: {self}")
    self.set_topic_ids()

    # Expose as RAG format
    if self.narrative_id is None:
        self.narrative_id = self.topic_id

autojudge_base.report.NeuclirReportSentence pydantic-model

Bases: BaseModel

Fields:

citations pydantic-field 

citations: Optional[List[str]] = None

text pydantic-field 

text: str

metadata pydantic-field 

metadata: Optional[Dict[str, Any]] = None

evaldata pydantic-field 

evaldata: Optional[Dict[str, Any]] = None

autojudge_base.report.RagtimeReportSentence pydantic-model

Bases: BaseModel

Fields:

citations pydantic-field 

citations: Optional[Dict[str, float]] = None

text pydantic-field 

text: str

metadata pydantic-field 

metadata: Optional[Dict[str, Any]] = None

evaldata pydantic-field 

evaldata: Optional[Dict[str, Any]] = None

autojudge_base.report.Rag24ReportSentence pydantic-model

Bases: BaseModel

Fields:

citations pydantic-field 

citations: Optional[List[int]] = None

text pydantic-field 

text: str

metadata pydantic-field 

metadata: Optional[Dict[str, Any]] = None

evaldata pydantic-field 

evaldata: Optional[Dict[str, Any]] = None

autojudge_base.report.RankedDocument pydantic-model

Bases: BaseModel

Fields:

doc_id pydantic-field 

doc_id: str

rank pydantic-field 

rank: int

doc pydantic-field 

doc: Optional[Document] = None

score pydantic-field 

score: Optional[float] = None

autojudge_base.report.RetrievedDocuments pydantic-model

Bases: BaseModel

Fields:

query_id pydantic-field 

query_id: str

test_collection pydantic-field 

test_collection: str

run_id pydantic-field 

run_id: Optional[str] = None

metadata pydantic-field 

metadata: Optional[Dict[str, Any]] = None

ranked_docs pydantic-field 

ranked_docs: List[RankedDocument]

Request

autojudge_base.request.Request pydantic-model

Bases: BaseModel

Fields:

request_id pydantic-field 

request_id: str

collection_ids pydantic-field 

collection_ids: Optional[List[str]] = None

background pydantic-field 

background: Optional[str] = None

original_background pydantic-field 

original_background: Optional[str] = None

problem_statement pydantic-field 

problem_statement: Optional[str] = None

limit pydantic-field 

limit: Optional[int] = None

word_limit pydantic-field 

word_limit: Optional[int] = None

title pydantic-field 

title: str

Document

autojudge_base.document.document.Document pydantic-model

Bases: BaseModel

NeuCLIR/RAGtime documents and translations.

Config:

  • extra: allow

Fields:

id pydantic-field 

id: str

text pydantic-field 

text: str

title pydantic-field 

title: Optional[str] = None

url pydantic-field 

url: Optional[str] = None

metadata pydantic-field 

metadata: Optional[Dict[str, Any]] = None

created pydantic-field 

created: Optional[str] = None

cc_file pydantic-field 

cc_file: Optional[str] = None

time pydantic-field 

time: Optional[str] = None

lang pydantic-field 

lang: Optional[str] = None

model_config class-attribute instance-attribute 

model_config = ConfigDict(extra='allow')

get_text 

get_text() -> str

All text of the document. Add title if defined.

Source code in src/autojudge_base/document/document.py 
36
37
38
39
40
41
42
43
def get_text(self) -> str:
    """All text of the document. Add title if defined."""

    clean_text = self.text
    if self.title is not None:
        return self.title+" "+clean_text
    else:        
        return clean_text

get_document_text 

get_document_text() -> str
Source code in src/autojudge_base/document/document.py 
45
46
def get_document_text(self) -> str:
    return self.get_text()

get_paragraphs 

get_paragraphs() -> List[str]
Source code in src/autojudge_base/document/document.py 
48
49
def get_paragraphs(self) ->List[str]:
    return get_paragraph_chunks(self.get_text())

get_sentences 

get_sentences() -> List[str]
Source code in src/autojudge_base/document/document.py 
51
52
def get_sentences(self) ->List[str]:
    return get_sentence_chunks_on_newline(self.get_text())

get_text_chunks 

get_text_chunks(limit: int) -> List[str]

Break full text into chunks up to limit, obeying sentence boundaries.

Source code in src/autojudge_base/document/document.py 
54
55
56
57
58
59
60
61
62
63
def get_text_chunks(self, limit:int) -> List[str]:
    """Break full text into chunks up to `limit`, obeying sentence boundaries."""
    if not self._full_text_chunks or limit != self._full_text_chunk_limit:
        self._full_text_chunks = get_limit_length_chunks(self.get_sentences() ,limit=limit)
        self._full_text_chunk_limit = limit

        if len(self._full_text_chunks)>1: 
            print(f"Breaking documents {self.id} into {len(self._full_text_chunks)} chunks of length {limit}")

    return self._full_text_chunks