Building a Chatbot Evaluation Framework: LLM-as-Judge, RAGAS, and Automated Testing Pipelines

from ragas import evaluate
from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_precision,
    context_recall,
)
from datasets import Dataset

# Prepare evaluation dataset
eval_data = {
    "question": [
        "What is the company's annual leave policy?",
        "How do I apply for remote work?",
        "How many days of family event leave are available?",
    ],
    "answer": [
        "Employees with over 1 year of service receive 15 days of annual leave. "
        "After 3 years of service, an additional day is added every 2 years.",
        "Remote work requires team lead approval followed by application "
        "through the HR system. Up to 3 days per week are allowed, "
        "with Monday and Friday being mandatory in-office days.",
        "Family event leave includes 5 days for marriage, "
        "10 days for spouse's childbirth, 5 days for parent's death, "
        "and 3 days for sibling's death.",
    ],
    "contexts": [
        [
            "Annual Leave Policy: Employees with over 1 year of service "
            "receive 15 paid annual leave days. After 3 years of service, "
            "1 additional day is added every 2 years. "
            "Unused leave does not carry over."
        ],
        [
            "Remote Work Guide: Employees wishing to work remotely must "
            "obtain prior approval from their team lead and apply through "
            "the HR portal. Up to 3 remote work days per week are allowed. "
            "Monday and Friday are mandatory in-office days for all employees."
        ],
        [
            "Family Event Leave: Marriage 5 days, spouse childbirth 10 days, "
            "parent death 5 days, grandparent death 3 days, "
            "sibling death 3 days."
        ],
    ],
    "ground_truth": [
        "15 days after 1 year, plus 1 additional day every 2 years after 3 years",
        "Team lead approval then HR system application, up to 3 days, Mon/Fri in-office required",
        "Marriage 5 days, spouse childbirth 10 days, parent death 5 days, sibling death 3 days",
    ],
}

dataset = Dataset.from_dict(eval_data)

# Run RAGAS evaluation
result = evaluate(
    dataset=dataset,
    metrics=[
        faithfulness,
        answer_relevancy,
        context_precision,
        context_recall,
    ],
)

print(result)
# Example results:
# faithfulness: 0.92
# answer_relevancy: 0.88
# context_precision: 0.95
# context_recall: 0.90

from ragas.metrics.base import MetricWithLLM
from dataclasses import dataclass, field

@dataclass
class EmpathyScore(MetricWithLLM):
    """Custom metric that evaluates empathy level in support responses (0-1)"""
    name: str = "empathy_score"
    evaluation_mode: str = "qa"

    async def _ascore(self, row, callbacks=None):
        prompt = (
            "Evaluate the following customer support response for "
            "appropriate empathetic expression on a scale of 0 to 1.\n\n"
            f"Question: {row['question']}\n"
            f"Answer: {row['answer']}\n\n"
            "Respond with only the numeric score."
        )
        response = await self.llm.agenerate_text(prompt)
        try:
            return float(response.generations[0][0].text.strip())
        except (ValueError, IndexError):
            return 0.0

import openai
import json
from typing import TypedDict

class EvalResult(TypedDict):
    score: int
    reasoning: str

def llm_as_judge_evaluate(
    question: str,
    answer: str,
    criteria: str,
    model: str = "gpt-4o",
) -> EvalResult:
    """Evaluate answer quality on a 1-5 scale using LLM-as-Judge"""

    system_prompt = """You are an expert judge evaluating AI chatbot response quality.
Rate the response on a 1-5 scale according to the given criteria and explain your reasoning.

Scoring rubric:
- 1: Completely inappropriate or incorrect response
- 2: Partially relevant but contains significant errors
- 3: Basically correct but has room for improvement
- 4: Good quality, meets most expectations
- 5: Excellent response, perfectly meets all criteria

You must respond in JSON format:
{"score": number, "reasoning": "evaluation reasoning"}"""

    user_prompt = f"""Evaluation criteria: {criteria}

User question: {question}

Chatbot response: {answer}

Evaluate the above response according to the criteria."""

    client = openai.OpenAI()
    response = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt},
        ],
        temperature=0.0,
        response_format={"type": "json_object"},
    )

    return json.loads(response.choices[0].message.content)


# Usage example
result = llm_as_judge_evaluate(
    question="What is the difference between a list and a tuple in Python?",
    answer="Lists are created with square brackets ([]) and are mutable. "
           "Tuples are created with parentheses (()) and are immutable. "
           "In terms of performance, tuples are slightly faster than lists.",
    criteria="Evaluate based on accuracy, completeness, and clarity",
)
print(f"Score: {result['score']}/5")
print(f"Reasoning: {result['reasoning']}")

def pairwise_compare(
    question: str,
    answer_a: str,
    answer_b: str,
    criteria: str,
    model: str = "gpt-4o",
) -> dict:
    """Compare two answers and select the better one"""

    system_prompt = """You are an expert judge comparing AI chatbot responses.
Compare responses A and B and determine which is better.

You must respond in JSON format:
{"winner": "A" or "B" or "tie", "reasoning": "comparison reasoning"}

Important: Do not be influenced by the order of responses.
Judge solely on content quality."""

    user_prompt = f"""Evaluation criteria: {criteria}

User question: {question}

Response A: {answer_a}

Response B: {answer_b}

Compare and evaluate both responses."""

    client = openai.OpenAI()
    response = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt},
        ],
        temperature=0.0,
        response_format={"type": "json_object"},
    )

    return json.loads(response.choices[0].message.content)

def debiased_pairwise_compare(
    question: str,
    answer_a: str,
    answer_b: str,
    criteria: str,
) -> dict:
    """Pairwise comparison with position bias mitigation"""

    # First evaluation: A presented first
    result_1 = pairwise_compare(question, answer_a, answer_b, criteria)

    # Second evaluation: B presented first (order reversed)
    result_2 = pairwise_compare(question, answer_b, answer_a, criteria)
    # Invert result_2's winner
    if result_2["winner"] == "A":
        result_2["winner"] = "B"
    elif result_2["winner"] == "B":
        result_2["winner"] = "A"

    # Aggregate results
    if result_1["winner"] == result_2["winner"]:
        return {
            "winner": result_1["winner"],
            "confidence": "high",
            "reasoning": f"Both evaluations agree: {result_1['reasoning']}",
        }
    else:
        return {
            "winner": "tie",
            "confidence": "low",
            "reasoning": (
                f"Evaluations disagree - "
                f"Forward: {result_1['winner']}, "
                f"Reversed: {result_2['winner']}"
            ),
        }

from ragas.testset.generator import TestsetGenerator
from ragas.testset.evolutions import simple, reasoning, multi_context
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_community.document_loaders import DirectoryLoader

# Load documents
loader = DirectoryLoader("./knowledge_base/", glob="**/*.md")
documents = loader.load()

# Configure test set generator
generator_llm = ChatOpenAI(model="gpt-4o")
critic_llm = ChatOpenAI(model="gpt-4o")
embeddings = OpenAIEmbeddings()

generator = TestsetGenerator.from_langchain(
    generator_llm=generator_llm,
    critic_llm=critic_llm,
    embeddings=embeddings,
)

# Generate test set with varied difficulty levels
testset = generator.generate_with_langchain_docs(
    documents=documents,
    test_size=200,
    distributions={
        simple: 0.4,        # Simple factual questions 40%
        reasoning: 0.3,     # Questions requiring reasoning 30%
        multi_context: 0.3,  # Questions needing multiple documents 30%
    },
)

# Export as DataFrame for review
df = testset.to_pandas()
df.to_csv("golden_dataset_draft.csv", index=False)
print(f"Generated test cases: {len(df)}")

# tests/test_chatbot_quality.py
import pytest
from deepeval import assert_test
from deepeval.test_case import LLMTestCase
from deepeval.metrics import (
    AnswerRelevancyMetric,
    FaithfulnessMetric,
    HallucinationMetric,
    GEval,
)

# Custom G-Eval metric: response tone evaluation
tone_metric = GEval(
    name="Professional Tone",
    criteria=(
        "Evaluate whether the response maintains a professional "
        "and courteous tone. It should not contain colloquialisms, "
        "emojis, or inappropriate expressions."
    ),
    evaluation_params=["actual_output"],
    threshold=0.7,
)

faithfulness_metric = FaithfulnessMetric(threshold=0.8)
relevancy_metric = AnswerRelevancyMetric(threshold=0.7)
hallucination_metric = HallucinationMetric(threshold=0.5)


@pytest.fixture
def chatbot_response():
    """Fixture to generate chatbot responses for testing"""
    from app.chatbot import get_response
    return get_response


class TestChatbotQuality:
    """Chatbot response quality regression tests"""

    def test_faq_faithfulness(self, chatbot_response):
        """Verify FAQ answers are faithful to retrieved context"""
        question = "How many annual leave days do I get?"
        response = chatbot_response(question)

        test_case = LLMTestCase(
            input=question,
            actual_output=response["answer"],
            retrieval_context=response["contexts"],
        )
        assert_test(test_case, [faithfulness_metric])

    def test_answer_relevancy(self, chatbot_response):
        """Verify answers are relevant to the question"""
        question = "How do I apply for remote work?"
        response = chatbot_response(question)

        test_case = LLMTestCase(
            input=question,
            actual_output=response["answer"],
        )
        assert_test(test_case, [relevancy_metric])

    def test_no_hallucination(self, chatbot_response):
        """Verify no hallucinations are present"""
        question = "How is severance pay calculated?"
        response = chatbot_response(question)

        test_case = LLMTestCase(
            input=question,
            actual_output=response["answer"],
            context=response["contexts"],
        )
        assert_test(test_case, [hallucination_metric])

    def test_professional_tone(self, chatbot_response):
        """Verify professional tone is maintained"""
        question = "When is payday?"
        response = chatbot_response(question)

        test_case = LLMTestCase(
            input=question,
            actual_output=response["answer"],
        )
        assert_test(test_case, [tone_metric])

# .github/workflows/chatbot-eval.yml
name: Chatbot Evaluation Pipeline

on:
  pull_request:
    paths:
      - 'prompts/**'
      - 'app/chatbot/**'
      - 'config/rag/**'

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: pip install -r requirements-eval.txt

      - name: Run RAGAS evaluation
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          python scripts/run_ragas_eval.py \
            --dataset golden_dataset.json \
            --output eval_results.json

      - name: Run DeepEval tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          deepeval test run tests/test_chatbot_quality.py \
            --verbose

      - name: Check regression thresholds
        run: |
          python scripts/check_thresholds.py \
            --results eval_results.json \
            --thresholds config/eval_thresholds.json

      - name: Post evaluation summary to PR
        if: always()
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const results = JSON.parse(
              fs.readFileSync('eval_results.json', 'utf8')
            );
            const body = `## Chatbot Evaluation Results
            | Metric | Score | Threshold | Status |
            |--------|-------|-----------|--------|
            | Faithfulness | ${results.faithfulness} | 0.85 | ${results.faithfulness >= 0.85 ? 'PASS' : 'FAIL'} |
            | Relevancy | ${results.relevancy} | 0.80 | ${results.relevancy >= 0.80 ? 'PASS' : 'FAIL'} |
            | Context Precision | ${results.context_precision} | 0.80 | ${results.context_precision >= 0.80 ? 'PASS' : 'FAIL'} |`;
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: body
            });