Evaluating Prompt Quality

[
  {
    "id": "1",
    "input": "Summarize: The Great Gatsby is a novel...",
    "expected_output": "A 1920s-set novel about wealth, love, and the American Dream",
    "category": "literature"
  },
  {
    "id": "2",
    "input": "Translate to Spanish: Hello, how are you?",
    "expected_output": "Hola, ¿cómo estás?",
    "category": "translation"
  },
  {
    "id": "3",
    "input": "Fix this code: def add(a, b return a + b",
    "expected_output": "def add(a, b):\n    return a + b",
    "category": "code"
  }
]

import anthropic
import json

judge_system = """
You are an expert evaluator of code quality.
Score code on:
1. Correctness (does it work?)
2. Readability (is it clear?)
3. Efficiency (is it optimal?)

Return a JSON object with:
- score (1-10)
- reasoning (brief explanation)
- issues (list of problems, or empty)
"""

client = anthropic.Anthropic()

def evaluate_code(code_to_review):
    judge_input = f"""
Evaluate this code:

```python
{code_to_review}
```

Return JSON with score, reasoning, and issues."""

    response = client.messages.create(
        model="claude-opus-4-6",
        max_tokens=500,
        system=judge_system,
        messages=[{
            "role": "user",
            "content": judge_input
        }]
    )

    # Parse the JSON response
    result_text = response.content[0].text
    try:
        result = json.loads(result_text)
        return result["score"]
    except:
        print("Failed to parse judge response")
        return None

from difflib import SequenceMatcher
import numpy as np

# Exact Match
def exact_match(output, expected):
    return 1.0 if output.strip() == expected.strip() else 0.0

# Sequence Similarity (for closeness)
def similarity_score(output, expected):
    return SequenceMatcher(None, output, expected).ratio()

# Usage:
output = "The Great Gatsby is about wealth and love in the 1920s"
expected = "A novel exploring American Dream and 1920s wealth"

score1 = exact_match(output, expected)  # 0.0
score2 = similarity_score(output, expected)  # 0.42

import anthropic
import json
from dataclasses import dataclass

client = anthropic.Anthropic()

@dataclass
class TestCase:
    id: str
    input: str
    expected_output: str

SYSTEM_PROMPT = """You are a code reviewer..."""

def evaluate_single_case(test_case: TestCase) -> dict:
    # Get model output
    response = client.messages.create(
        model="claude-opus-4-6",
        max_tokens=500,
        system=SYSTEM_PROMPT,
        messages=[{
            "role": "user",
            "content": test_case.input
        }]
    )
    actual_output = response.content[0].text

    # Score using LLM-as-judge
    judge_prompt = f"""
Input: {test_case.input}
Expected: {test_case.expected_output}
Actual: {actual_output}

Score accuracy 1-10."""

    judge_response = client.messages.create(
        model="claude-opus-4-6",
        max_tokens=100,
        messages=[{
            "role": "user",
            "content": judge_prompt
        }]
    )
    judge_output = judge_response.content[0].text

    return {
        "test_id": test_case.id,
        "actual_output": actual_output,
        "judge_score": judge_output
    }

def run_eval_pipeline(test_cases: list[TestCase]) -> dict:
    results = []
    scores = []

    for test_case in test_cases:
        result = evaluate_single_case(test_case)
        results.append(result)
        # Extract numeric score (simplified)
        score = int(result["judge_score"][0]) if result["judge_score"][0].isdigit() else 5
        scores.append(score)

    return {
        "total_tests": len(test_cases),
        "average_score": sum(scores) / len(scores),
        "min_score": min(scores),
        "max_score": max(scores),
        "results": results
    }

# Usage:
test_data = [
    TestCase("1", "Review: def x(): pass", "Good but lacks docstring"),
    # ...
]
report = run_eval_pipeline(test_data)
print(json.dumps(report, indent=2))

scores = {
    "v1_baseline": 65,
    "v2_better_system_prompt": 72,
    "v3_added_examples": 78,
    "v4_clarified_format": 81
}

for version, score in scores.items():
    improvement = score - 65
    print(f"{version}: {score}/100 (+{improvement})")