25 lines
882 B
Python
25 lines
882 B
Python
import json
|
|
from pathlib import Path
|
|
from pydantic import BaseModel, Field
|
|
|
|
TEST_FILE = str(Path(__file__).parent / "tests.jsonl")
|
|
|
|
|
|
class TestQuestion(BaseModel):
|
|
"""A test question with expected keywords and reference answer."""
|
|
|
|
question: str = Field(description="The question to ask the RAG system")
|
|
keywords: list[str] = Field(description="Keywords that must appear in retrieved context")
|
|
reference_answer: str = Field(description="The reference answer for this question")
|
|
category: str = Field(description="Question category (e.g., direct_fact, spanning, temporal)")
|
|
|
|
|
|
def load_tests() -> list[TestQuestion]:
|
|
"""Load test questions from JSONL file."""
|
|
tests = []
|
|
with open(TEST_FILE, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
data = json.loads(line.strip())
|
|
tests.append(TestQuestion(**data))
|
|
return tests
|