#!/usr/bin/env python3
"""RAG evaluation script using RAGAS metrics.

Measures: faithfulness, answer relevancy, context precision, context recall.
Run this against your RAG pipeline's output to quantify where it fails.

Usage: python ragas_eval.py <eval_samples.jsonl>

Requires: pip install ragas
Set OPENAI_API_KEY in env for LLM-as-judge evaluations.
"""
from __future__ import annotations

import json
import sys
from dataclasses import dataclass


@dataclass
class EvalSample:
    question: str
    answer: str
    contexts: list[str]
    reference: str = ""


def load_samples(path: str) -> list[EvalSample]:
    """Load eval set from JSONL: {question, answer, contexts, [reference]}."""
    samples = []
    with open(path) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            obj = json.loads(line)
            samples.append(EvalSample(
                question=obj["question"],
                answer=obj["answer"],
                contexts=obj.get("contexts", []),
                reference=obj.get("reference", ""),
            ))
    return samples


def evaluate_ragas(samples: list[EvalSample]) -> dict[str, float]:
    """Run RAGAS metrics. Requires OPENAI_API_KEY."""
    try:
        from ragas import EvaluationDataset, SingleTurnSample, evaluate
        from ragas.metrics import (
            faithfulness, answer_relevancy,
            context_precision, context_recall,
        )
    except ImportError:
        print("Install ragas: pip install ragas")
        sys.exit(1)

    ragas_samples = [
        SingleTurnSample(
            user_input=s.question,
            response=s.answer,
            retrieved_contexts=s.contexts,
            reference=s.reference,
        )
        for s in samples
    ]
    dataset = EvaluationDataset(samples=ragas_samples)
    metrics = [faithfulness, answer_relevancy, context_recall, context_precision]
    result = evaluate(dataset=dataset, metrics=metrics)

    return {name: round(float(val), 4) for name, val in result.items()}


def main() -> None:
    if len(sys.argv) < 2:
        print("Usage: python ragas_eval.py <eval_samples.jsonl>")
        print("\nJSONL format per line:")
        print('  {"question":"...","answer":"...","contexts":[...],"reference":"..."}')
        sys.exit(1)

    samples = load_samples(sys.argv[1])
    print(f"Loaded {len(samples)} samples")

    if len(samples) == 0:
        print("No samples to evaluate.")
        return

    scores = evaluate_ragas(samples)

    print("\n=== RAGAS Metrics ===")
    for metric, value in scores.items():
        bar = "#" * int(value * 20)
        print(f"  {metric:25s}: {value:.4f}  {bar}")

    # Diagnostic hints
    print("\n=== Diagnostic ===")
    issues = []
    faithfulness_val = scores.get("faithfulness", 0)
    context_recall_val = scores.get("context_recall", 0)
    answer_rel = scores.get("answer_relevancy", 0)
    context_prec = scores.get("context_precision", 0)

    if faithfulness_val < 0.7:
        issues.append("Low faithfulness -> generator/prompt problem: model fabricating")
    if answer_rel < 0.7:
        issues.append("Low answer relevance -> model padding or evading the question")
    if context_recall_val < 0.7:
        issues.append("Low context recall -> retriever missing needed chunks")
    if context_prec < 0.7:
        issues.append("Low context precision -> too much irrelevant retrieved text")

    if issues:
        for msg in issues:
            print(f"  ! {msg}")
    else:
        print("  All metrics above 0.7 -- RAG pipeline healthy")

    # Threshold check (exit code for CI)
    threshold = 0.6
    failing = [m for m, v in scores.items() if v < threshold]
    if failing:
        print(f"\nFAIL: {failing} below threshold {threshold}")
        sys.exit(1)


if __name__ == "__main__":
    main()