lambdagent/agentexample/qaagent67/scripts/analyze_results.py

#!/usr/bin/env python3
"""
Step 3: Analyze RAG vs Wiki comparison results
"""
import json
import sys
from collections import Counter, defaultdict
from pathlib import Path

sys.path.insert(0, '/home/67/knowledge/maritime')
sys.path.insert(0, '/home/67/lambdagentpaas')
# lambdagent migrated to src-layout — add the src dir so 'from lambdagent.X'
# still resolves when running outside an editable-install venv.
sys.path.insert(0, '/home/67/lambdagentpaas/lambdagent/src')

RESULTS_FILE = Path('/home/67/knowledge/maritime/batch_results_200.json')
REPORT_FILE = Path('/home/67/knowledge/maritime/analysis_report.md')

from lambdagent.providers import create_provider

def main():
    with open(RESULTS_FILE) as f:
        data = json.load(f)

    results = data['results']
    meta = data.get('meta', {})
    total = len(results)

    print(f"Analyzing {total} Q&A pairs...")

    # ── Basic Stats ──
    rag_times = [r['rag']['elapsed'] for r in results if not r['rag'].get('error')]
    wiki_times = [r['wiki']['elapsed'] for r in results if not r['wiki'].get('error')]
    rag_lens = [len(r['rag']['answer']) for r in results if not r['rag'].get('error')]
    wiki_lens = [len(r['wiki']['answer']) for r in results if not r['wiki'].get('error')]

    rag_errors = sum(1 for r in results if r['rag'].get('error'))
    wiki_errors = sum(1 for r in results if r['wiki'].get('error'))

    # Wiki source types
    wiki_sources = Counter(r['wiki'].get('source_type', 'unknown') for r in results)

    # By question type
    by_type = defaultdict(lambda: {'rag_times': [], 'wiki_times': [], 'rag_lens': [], 'wiki_lens': [], 'count': 0})
    for r in results:
        t = r.get('type', 'unknown')
        by_type[t]['count'] += 1
        if not r['rag'].get('error'):
            by_type[t]['rag_times'].append(r['rag']['elapsed'])
            by_type[t]['rag_lens'].append(len(r['rag']['answer']))
        if not r['wiki'].get('error'):
            by_type[t]['wiki_times'].append(r['wiki']['elapsed'])
            by_type[t]['wiki_lens'].append(len(r['wiki']['answer']))

    # By difficulty
    by_diff = defaultdict(lambda: {'rag_times': [], 'wiki_times': [], 'count': 0})
    for r in results:
        d = r.get('difficulty', 'medium')
        by_diff[d]['count'] += 1
        if not r['rag'].get('error'):
            by_diff[d]['rag_times'].append(r['rag']['elapsed'])
        if not r['wiki'].get('error'):
            by_diff[d]['wiki_times'].append(r['wiki']['elapsed'])

    # ── Quality Analysis using LLM ──
    # Sample 20 questions for LLM-based quality evaluation
    print("Running LLM quality evaluation on 20 sampled answers...")
    provider = create_provider('ollama', model='qwen2.5:32b', timeout=600)

    import random
    sample = random.sample(results, min(20, total))

    quality_scores = {'rag': [], 'wiki': []}
    for i, r in enumerate(sample):
        eval_prompt = f"""请评估以下两个回答的质量。评分标准(1-5分):
- 准确性: 信息是否正确
- 完整性: 是否覆盖问题要点
- 引用: 是否有来源标注
- 条理性: 结构是否清晰

问题: {r['question']}

回答A (RAG模式):
{r['rag']['answer'][:800]}

回答B (Wiki模式):
{r['wiki']['answer'][:800]}

请严格按以下JSON格式输出评分，不要输出其他内容:
{{"rag_score": <1-5>, "wiki_score": <1-5>, "rag_strength": "<一句话>", "wiki_strength": "<一句话>", "winner": "<rag或wiki或tie>"}}"""

        try:
            result = provider.chat([{'role': 'user', 'content': eval_prompt}])
            # Try to parse JSON from response
            import re
            json_match = re.search(r'\{[^{}]+\}', result)
            if json_match:
                scores = json.loads(json_match.group())
                quality_scores['rag'].append(scores.get('rag_score', 3))
                quality_scores['wiki'].append(scores.get('wiki_score', 3))
                print(f"  [{i+1}/20] RAG={scores.get('rag_score')}, Wiki={scores.get('wiki_score')}, Winner={scores.get('winner')}")
            else:
                print(f"  [{i+1}/20] Could not parse scores")
        except Exception as e:
            print(f"  [{i+1}/20] Error: {e}")

    # ── Generate Report ──
    def avg(lst):
        return round(sum(lst) / len(lst), 1) if lst else 0
    def median(lst):
        if not lst: return 0
        s = sorted(lst)
        n = len(s)
        return s[n//2] if n % 2 else round((s[n//2-1] + s[n//2]) / 2, 1)

    rag_avg_score = avg(quality_scores['rag']) if quality_scores['rag'] else '-'
    wiki_avg_score = avg(quality_scores['wiki']) if quality_scores['wiki'] else '-'

    report = f"""# Maritime QA Comparison Report
# RAG vs Wiki -- {total} Questions

Generated: {meta.get('completed_at', 'N/A')}
Total time: {meta.get('total_minutes', 'N/A')} minutes

---

## 1. Overview

| Metric | RAG | Wiki |
|--------|-----|------|
| Questions answered | {total - rag_errors} | {total - wiki_errors} |
| Errors | {rag_errors} | {wiki_errors} |
| Avg response time | {avg(rag_times)}s | {avg(wiki_times)}s |
| Median response time | {median(rag_times)}s | {median(wiki_times)}s |
| Min/Max time | {min(rag_times) if rag_times else '-'}/{max(rag_times) if rag_times else '-'}s | {min(wiki_times) if wiki_times else '-'}/{max(wiki_times) if wiki_times else '-'}s |
| Avg answer length | {avg(rag_lens)} chars | {avg(wiki_lens)} chars |
| Total time | {sum(rag_times):.0f}s | {sum(wiki_times):.0f}s |

## 2. Quality Evaluation (LLM-judged, 20 sample)

| Metric | RAG | Wiki |
|--------|-----|------|
| Average score (1-5) | {rag_avg_score} | {wiki_avg_score} |
| Scores distribution | {Counter(quality_scores['rag'])} | {Counter(quality_scores['wiki'])} |

## 3. By Question Type

| Type | Count | RAG avg time | Wiki avg time | RAG avg len | Wiki avg len |
|------|-------|-------------|--------------|------------|-------------|
"""
    for t in ['fact', 'reasoning', 'synthesis']:
        if t in by_type:
            d = by_type[t]
            report += f"| {t} | {d['count']} | {avg(d['rag_times'])}s | {avg(d['wiki_times'])}s | {avg(d['rag_lens'])} | {avg(d['wiki_lens'])} |\n"

    report += f"""
## 4. By Difficulty

| Difficulty | Count | RAG avg time | Wiki avg time |
|-----------|-------|-------------|--------------|
"""
    for d_name in ['easy', 'medium', 'hard']:
        if d_name in by_diff:
            d = by_diff[d_name]
            report += f"| {d_name} | {d['count']} | {avg(d['rag_times'])}s | {avg(d['wiki_times'])}s |\n"

    report += f"""
## 5. Wiki Knowledge Accumulation

| Source Type | Count | Percentage |
|------------|-------|-----------|
"""
    for src, cnt in wiki_sources.most_common():
        report += f"| {src} | {cnt} | {cnt*100/total:.1f}% |\n"

    report += f"""
## 6. Key Findings

### RAG Mode Characteristics
- Average response time: {avg(rag_times)}s
- Direct retrieval from 62,011 text chunks
- Each answer based on top-{results[0]['rag'].get('chunks_used', 5) if results else 5} retrieved chunks
- Consistent response time regardless of question complexity

### Wiki Mode Characteristics
- Average response time: {avg(wiki_times)}s
- First query requires compilation (2 LLM calls), subsequent queries faster if wiki hit
- Wiki pages accumulated: {wiki_sources.get('wiki', 0)} hits / {wiki_sources.get('compiled', 0)} compilations
- Knowledge accumulates over time -- later questions benefit from earlier compilations

### Speed Comparison
- RAG is {'faster' if avg(rag_times) < avg(wiki_times) else 'slower'} by {abs(avg(rag_times) - avg(wiki_times))}s on average
- Wiki first-query overhead: ~{avg(wiki_times) - avg(rag_times):.0f}s extra for compilation

### Quality Comparison (LLM-judged)
- RAG average score: {rag_avg_score}/5
- Wiki average score: {wiki_avg_score}/5
"""

    with open(REPORT_FILE, 'w', encoding='utf-8') as f:
        f.write(report)

    print(f"\nReport saved to: {REPORT_FILE}")
    print("\n" + report)

if __name__ == '__main__':
    main()