lambdagent/agentexample/qaagent67lambda/scripts/test_finance_180.py

#!/usr/bin/env python3
"""
Finance domain test: 180 hand-written questions
Covers: fact, process, person, data, compare, temporal, synthesis
"""
import json
import sys
import time
import os

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
os.environ.setdefault('OLLAMA_EMBED_URL', 'http://127.0.0.1:11435/api/embed')

from search_engine import search as bm25_search
from search_unified import search as unified_search

QUESTIONS = [
    # ══════════════════════════════════════
    # FACT (35) — 具体数据/标准/规定
    # ══════════════════════════════════════
    {"q": "一般性公务接待用餐标准是多少", "type": "fact", "kw": ["120", "元"]},
    {"q": "南京大学纳税人识别号是多少", "type": "fact", "kw": ["12100000466007458"]},
    {"q": "会议费收入的税率是多少", "type": "fact", "kw": ["6%"]},
    {"q": "重要公务接待用餐标准是多少", "type": "fact", "kw": ["150", "元"]},
    {"q": "助研费每人每年上限是多少", "type": "fact", "kw": ["6万", "六万"]},
    {"q": "硕士研究生助研费每月上限", "type": "fact", "kw": ["600"]},
    {"q": "博士研究生助研费每月上限", "type": "fact", "kw": ["1000"]},
    {"q": "公务卡的有效期是多久", "type": "fact", "kw": ["公务卡"]},
    {"q": "财务处工作时间是什么", "type": "fact", "kw": ["工作时间"]},
    {"q": "财务处电话号码是多少", "type": "fact", "kw": ["025", "电话"]},
    {"q": "竞争性经费的定义是什么", "type": "fact", "kw": ["竞争"]},
    {"q": "非竞争性经费包括哪些", "type": "fact", "kw": ["非竞争"]},
    {"q": "外专接待餐费标准是多少", "type": "fact", "kw": ["外专", "餐"]},
    {"q": "培训费综合定额标准是多少", "type": "fact", "kw": ["培训", "定额"]},
    {"q": "全国知名专家咨询费标准", "type": "fact", "kw": ["专家", "咨询"]},
    {"q": "国内差旅伙食补助标准", "type": "fact", "kw": ["100", "伙食"]},
    {"q": "西藏青海新疆伙食补助标准", "type": "fact", "kw": ["120", "西藏"]},
    {"q": "市内交通费包干标准是多少", "type": "fact", "kw": ["交通费"]},
    {"q": "一类人员住宿费标准", "type": "fact", "kw": ["住宿", "一类"]},
    {"q": "设备费单价多少以上需要建账", "type": "fact", "kw": ["1000", "建账"]},
    {"q": "合同审签的金额门槛是多少", "type": "fact", "kw": ["合同", "万"]},
    {"q": "预约单有效期是多久", "type": "fact", "kw": ["预约"]},
    {"q": "财务自助平台的网址是什么", "type": "fact", "kw": ["网址", "平台"]},
    {"q": "科研间接经费的比例是多少", "type": "fact", "kw": ["间接", "比例"]},
    {"q": "横向经费管理费提取比例", "type": "fact", "kw": ["横向", "管理费"]},
    {"q": "出国人员每天伙食补助标准", "type": "fact", "kw": ["出国", "伙食"]},
    {"q": "图书单价多少以上需要建账", "type": "fact", "kw": ["图书", "1000"]},
    {"q": "批量图书多少以上需要建账", "type": "fact", "kw": ["批量", "10000"]},
    {"q": "差旅费报销的城市间交通标准", "type": "fact", "kw": ["城市间", "交通"]},
    {"q": "接待陪餐人数规定", "type": "fact", "kw": ["陪餐"]},
    {"q": "助研费发放日期是每月几号", "type": "fact", "kw": ["10日"]},
    {"q": "横向经费可以支出哪些费用", "type": "fact", "kw": ["横向"]},
    {"q": "纵向经费直接费用包括哪些", "type": "fact", "kw": ["直接"]},
    {"q": "会议综合定额外费用有哪些", "type": "fact", "kw": ["综合定额"]},
    {"q": "租车费用多少以上需要合同", "type": "fact", "kw": ["租车", "合同"]},

    # ══════════════════════════════════════
    # PROCESS (35) — 流程/能不能/怎么办
    # ══════════════════════════════════════
    {"q": "超标的餐费可以报销吗", "type": "process", "kw": ["不予报销"]},
    {"q": "横向经费可以报销超级经济舱吗", "type": "process", "kw": ["横向", "经济舱"]},
    {"q": "1250元的图书用竞争经费需要建账吗", "type": "process", "kw": ["建账", "1000"]},
    {"q": "纵向科研经费可以报销手机话费吗", "type": "process", "kw": ["手机", "话费"]},
    {"q": "论文查重费可以从直接经费列支吗", "type": "process", "kw": ["查重", "直接"]},
    {"q": "已提交的报销单可以自己退回吗", "type": "process", "kw": ["退回", "撤回"]},
    {"q": "助研费超过六万怎么扣税", "type": "process", "kw": ["六万", "扣税"]},
    {"q": "项目负责人也是财务一支笔需要别人签字吗", "type": "process", "kw": ["签字", "审批"]},
    {"q": "出差期间的网约车报销需要行程单吗", "type": "process", "kw": ["网约车"]},
    {"q": "外专劳务可以打给经办人吗", "type": "process", "kw": ["外专", "经办"]},
    {"q": "助研费线上单子提交后可以撤回吗", "type": "process", "kw": ["撤回", "撤销"]},
    {"q": "怎样查询之前报销的凭证", "type": "process", "kw": ["查询", "凭证"]},
    {"q": "助研费单据未交财务处想作废怎么取消", "type": "process", "kw": ["作废", "取消"]},
    {"q": "横向经费报销接待餐费需要邀请函吗", "type": "process", "kw": ["邀请函"]},
    {"q": "科研绩效额度超支预约单可以修改吗", "type": "process", "kw": ["修改"]},
    {"q": "如何办理外汇用汇手续", "type": "process", "kw": ["外汇"]},
    {"q": "如何开具或预借发票", "type": "process", "kw": ["发票", "预借"]},
    {"q": "差旅费网上报销操作步骤", "type": "process", "kw": ["差旅", "报销"]},
    {"q": "会议费报销需要哪些材料", "type": "process", "kw": ["会议费", "材料"]},
    {"q": "出国费用报销流程", "type": "process", "kw": ["出国"]},
    {"q": "用间接经费可以支出学生活动盒饭吗", "type": "process", "kw": ["间接", "盒饭"]},
    {"q": "保险费用可以报销吗", "type": "process", "kw": ["保险"]},
    {"q": "超标餐费发票可以按标准报销吗", "type": "process", "kw": ["超标"]},
    {"q": "火车票发票抬头错了能报销吗", "type": "process", "kw": ["抬头"]},
    {"q": "国补的发票可以报销吗", "type": "process", "kw": ["国补"]},
    {"q": "与专利相关的滞纳金可以报销吗", "type": "process", "kw": ["专利", "滞纳金"]},
    {"q": "国社科经费可以报销电脑吗", "type": "process", "kw": ["国社科", "电脑"]},
    {"q": "直接经费可以购买办公用品吗", "type": "process", "kw": ["直接", "办公"]},
    {"q": "在校研究生可以作为借款责任人吗", "type": "process", "kw": ["借款"]},
    {"q": "退休老师发放劳务费怎么预约", "type": "process", "kw": ["退休", "劳务"]},
    {"q": "自驾出行住宿费可以报销吗", "type": "process", "kw": ["自驾", "住宿"]},
    {"q": "合同注明有稿酬还能报销出版费吗", "type": "process", "kw": ["稿酬", "出版"]},
    {"q": "党费出差可以领取伙食补贴吗", "type": "process", "kw": ["党费"]},
    {"q": "研究生参与导师项目可以按校外劳务发放吗", "type": "process", "kw": ["研究生", "校外"]},
    {"q": "已办理资产建账可以跨年吗", "type": "process", "kw": ["资产", "跨年"]},

    # ══════════════════════════════════════
    # PERSON (20) — 找人/联系方式/部门
    # ══════════════════════════════════════
    {"q": "核算外汇打什么电话", "type": "person", "kw": ["电话", "025"]},
    {"q": "核销外汇找哪位老师办理", "type": "person", "kw": ["外汇"]},
    {"q": "修改校外老师人员信息找谁", "type": "person", "kw": ["修改", "人员"]},
    {"q": "横向经费管理归哪个科室", "type": "person", "kw": ["横向", "科室"]},
    {"q": "差旅费报销咨询找谁", "type": "person", "kw": ["差旅"]},
    {"q": "助研费发放问题联系方式", "type": "person", "kw": ["助研"]},
    {"q": "公务卡办理找哪个部门", "type": "person", "kw": ["公务卡"]},
    {"q": "苏州校区财务处联系方式", "type": "person", "kw": ["苏州"]},
    {"q": "鼓楼校区财务处在哪里", "type": "person", "kw": ["鼓楼"]},
    {"q": "仙林校区财务处地址", "type": "person", "kw": ["仙林"]},
    {"q": "外专费用审批找谁", "type": "person", "kw": ["外专"]},
    {"q": "科研经费管理归哪个部门", "type": "person", "kw": ["科研"]},
    {"q": "票据领取在哪里", "type": "person", "kw": ["票据"]},
    {"q": "发票开具找哪个窗口", "type": "person", "kw": ["发票"]},
    {"q": "人事代理人员开通财务平台权限找谁", "type": "person", "kw": ["人事代理"]},
    {"q": "自助投单机在哪里", "type": "person", "kw": ["投单机"]},
    {"q": "财务处处长是谁", "type": "person", "kw": ["处长"]},
    {"q": "会计核算科的职责", "type": "person", "kw": ["核算科"]},
    {"q": "预算管理科的职责", "type": "person", "kw": ["预算"]},
    {"q": "计算机软件管理部门是哪里", "type": "person", "kw": ["软件", "管理"]},

    # ══════════════════════════════════════
    # DATA (20) — 表格/标准表/费率表
    # ══════════════════════════════════════
    {"q": "学生劳务费标准", "type": "data", "kw": ["劳务"]},
    {"q": "各类人员住宿费限额标准", "type": "data", "kw": ["住宿", "限额"]},
    {"q": "各类人员交通工具乘坐标准", "type": "data", "kw": ["交通", "标准"]},
    {"q": "会议费综合定额标准表", "type": "data", "kw": ["会议", "定额"]},
    {"q": "培训费综合定额标准表", "type": "data", "kw": ["培训", "定额"]},
    {"q": "因公出国伙食补助标准表", "type": "data", "kw": ["出国", "伙食"]},
    {"q": "因公出国住宿费标准表", "type": "data", "kw": ["出国", "住宿"]},
    {"q": "接待费用餐标准对照表", "type": "data", "kw": ["接待", "标准"]},
    {"q": "南京地区住宿费标准", "type": "data", "kw": ["南京", "住宿"]},
    {"q": "北京地区住宿费标准", "type": "data", "kw": ["北京", "住宿"]},
    {"q": "科研经费间接费用提取比例表", "type": "data", "kw": ["间接", "比例"]},
    {"q": "纵向科研经费绩效支出比例", "type": "data", "kw": ["绩效", "比例"]},
    {"q": "横向经费管理费提取比例表", "type": "data", "kw": ["横向", "管理费"]},
    {"q": "各经费类型报销审批权限对照", "type": "data", "kw": ["审批", "权限"]},
    {"q": "公务卡还款时间表", "type": "data", "kw": ["公务卡", "还款"]},
    {"q": "印花税税率表", "type": "data", "kw": ["印花税"]},
    {"q": "个人所得税税率表", "type": "data", "kw": ["个人所得税"]},
    {"q": "增值税税率分类", "type": "data", "kw": ["增值税"]},
    {"q": "各院系财务负责人名单", "type": "data", "kw": ["负责人"]},
    {"q": "二类人员有哪些", "type": "data", "kw": ["二类"]},

    # ══════════════════════════════════════
    # COMPARE (15) — 对比
    # ══════════════════════════════════════
    {"q": "竞争性经费和非竞争性经费的区别", "type": "compare", "kw": ["竞争", "非竞争"]},
    {"q": "直接经费和间接经费有什么不同", "type": "compare", "kw": ["直接", "间接"]},
    {"q": "纵向经费和横向经费的管理区别", "type": "compare", "kw": ["纵向", "横向"]},
    {"q": "一类人员和二类人员的标准区别", "type": "compare", "kw": ["一类", "二类"]},
    {"q": "工作餐和接待餐的标准有什么不同", "type": "compare", "kw": ["工作餐", "接待"]},
    {"q": "国库经费和非国库经费的区别", "type": "compare", "kw": ["国库"]},
    {"q": "校外劳务和校内劳务的发放方式区别", "type": "compare", "kw": ["校外", "校内"]},
    {"q": "会议费和培训费有什么区别", "type": "compare", "kw": ["会议费", "培训费"]},
    {"q": "设备费和材料费的区别", "type": "compare", "kw": ["设备", "材料"]},
    {"q": "出国经费和国内差旅经费的管理区别", "type": "compare", "kw": ["出国", "国内"]},
    {"q": "科研助理和研究生助研费的发放区别", "type": "compare", "kw": ["科研助理", "助研"]},
    {"q": "预算调整和预算追加有什么区别", "type": "compare", "kw": ["预算"]},
    {"q": "在线报销和线下报销的区别", "type": "compare", "kw": ["在线", "线下"]},
    {"q": "公务卡和个人银行卡报销的区别", "type": "compare", "kw": ["公务卡"]},
    {"q": "日常报销和差旅报销有什么不同", "type": "compare", "kw": ["日常", "差旅"]},

    # ══════════════════════════════════════
    # TEMPORAL (15) — 时效性
    # ══════════════════════════════════════
    {"q": "财务服务指南最新版本是哪年的", "type": "temporal", "kw": ["2024"]},
    {"q": "差旅费管理办法最新修订时间", "type": "temporal", "kw": ["差旅"]},
    {"q": "横向经费管理办法什么时候修订的", "type": "temporal", "kw": ["横向"]},
    {"q": "助研费发放政策近年有什么变化", "type": "temporal", "kw": ["助研"]},
    {"q": "科研经费管理改革最新政策", "type": "temporal", "kw": ["科研", "改革"]},
    {"q": "增值税税率近年有什么调整", "type": "temporal", "kw": ["增值税"]},
    {"q": "个人所得税起征点是多少", "type": "temporal", "kw": ["起征点"]},
    {"q": "公务卡政策最新要求", "type": "temporal", "kw": ["公务卡"]},
    {"q": "预算绩效管理最新规定", "type": "temporal", "kw": ["绩效"]},
    {"q": "年度个税申报截止时间", "type": "temporal", "kw": ["个税", "截止"]},
    {"q": "助研费发放时间有变化吗", "type": "temporal", "kw": ["助研", "时间"]},
    {"q": "财务处自助平台什么时候上线的", "type": "temporal", "kw": ["平台"]},
    {"q": "学生助学贷款一般几月份到账", "type": "temporal", "kw": ["助学贷款"]},
    {"q": "预借发票核销期限是多久", "type": "temporal", "kw": ["预借", "核销"]},
    {"q": "支付金额多少以上需要处长签字", "type": "temporal", "kw": ["处长", "签字"]},

    # ══════════════════════════════════════
    # SYNTHESIS (20) — 综合分析
    # ══════════════════════════════════════
    {"q": "总结南京大学差旅费报销的完整流程", "type": "synthesis", "kw": ["差旅", "流程"]},
    {"q": "概述南京大学科研经费管理体系", "type": "synthesis", "kw": ["科研", "管理"]},
    {"q": "梳理南京大学会议费报销的完整规定", "type": "synthesis", "kw": ["会议费"]},
    {"q": "总结横向科研经费的使用规范", "type": "synthesis", "kw": ["横向"]},
    {"q": "概述助研费发放的完整制度", "type": "synthesis", "kw": ["助研"]},
    {"q": "梳理外专费用报销的全部要求", "type": "synthesis", "kw": ["外专"]},
    {"q": "总结培训费报销的标准和流程", "type": "synthesis", "kw": ["培训"]},
    {"q": "概述接待费报销的完整规定", "type": "synthesis", "kw": ["接待"]},
    {"q": "梳理资产建账的完整流程", "type": "synthesis", "kw": ["资产", "建账"]},
    {"q": "总结公务卡使用和管理规定", "type": "synthesis", "kw": ["公务卡"]},
    {"q": "概述财务处各科室的职能分工", "type": "synthesis", "kw": ["科室"]},
    {"q": "梳理出国经费报销的完整规定", "type": "synthesis", "kw": ["出国"]},
    {"q": "总结票据管理的相关规定", "type": "synthesis", "kw": ["票据"]},
    {"q": "概述预算管理的主要流程", "type": "synthesis", "kw": ["预算"]},
    {"q": "梳理纵向科研经费结题后的经费处理", "type": "synthesis", "kw": ["结题"]},
    {"q": "总结财务报销需要注意的常见问题", "type": "synthesis", "kw": ["报销", "注意"]},
    {"q": "概述维修改造工程的财务报销流程", "type": "synthesis", "kw": ["维修", "工程"]},
    {"q": "梳理劳务费发放的各类标准", "type": "synthesis", "kw": ["劳务"]},
    {"q": "总结网上报销系统的使用方法", "type": "synthesis", "kw": ["网上", "系统"]},
    {"q": "概述财务自助平台的各项功能", "type": "synthesis", "kw": ["平台"]},
]

print(f"{'='*60}")
print(f"  Finance 180-Question Test (LambdaRAG v3.1)")
print(f"{'='*60}")

# Count by type
from collections import Counter
type_counts = Counter(q['type'] for q in QUESTIONS)
print(f"Distribution: {dict(type_counts)}")
print(f"Total: {len(QUESTIONS)}")

results = []
for i, q in enumerate(QUESTIONS):
    t0 = time.time()
    try:
        bm25_r = bm25_search(q['q'], top_k=5)
    except:
        bm25_r = []
    bm25_time = time.time() - t0

    t0 = time.time()
    try:
        uni_r = unified_search(q['q'], top_k=5)
    except:
        uni_r = []
    uni_time = time.time() - t0

    def check(res, kws):
        if not res: return 0, []
        txt = ' '.join([r.get('text','') + ' ' + r.get('source','') for r in res])
        found = [k for k in kws if k in txt]
        return len(found) / max(len(kws), 1), found

    bm25_sc, _ = check(bm25_r, q['kw'])
    uni_sc, _ = check(uni_r, q['kw'])
    winner = 'unified' if uni_sc > bm25_sc else ('tie' if uni_sc == bm25_sc else 'bm25')
    qtype = uni_r[0].get('_query_type', '?') if uni_r else '?'

    if winner != 'tie' or (i+1) % 30 == 0:
        print(f"Q{i+1:3d} [{q['type']:9s}] B={bm25_sc:.0%} U={uni_sc:.0%} {winner:7s} | {q['q'][:40]}")

    results.append({
        'id': i+1, 'question': q['q'], 'expected_type': q['type'],
        'classified_type': qtype,
        'bm25_relevance': round(bm25_sc, 2),
        'unified_relevance': round(uni_sc, 2),
        'bm25_time': round(bm25_time, 2),
        'unified_time': round(uni_time, 2),
        'winner': winner,
    })

# Summary
print(f"\n{'='*60}")
print("SUMMARY")
print(f"{'='*60}")
bm25_avg = sum(r['bm25_relevance'] for r in results) / len(results)
uni_avg = sum(r['unified_relevance'] for r in results) / len(results)
print(f"\nOverall: BM25={bm25_avg:.1%}  Unified={uni_avg:.1%}")
wins = {'unified': 0, 'bm25': 0, 'tie': 0}
for r in results: wins[r['winner']] += 1
print(f"Wins: U={wins['unified']} B={wins['bm25']} Tie={wins['tie']}")

print(f"\nBy type:")
for t in ['fact','process','person','data','compare','temporal','synthesis']:
    tr = [r for r in results if r['expected_type'] == t]
    if not tr: continue
    b = sum(r['bm25_relevance'] for r in tr) / len(tr)
    u = sum(r['unified_relevance'] for r in tr) / len(tr)
    wu = sum(1 for r in tr if r['winner'] == 'unified')
    wb = sum(1 for r in tr if r['winner'] == 'bm25')
    print(f"  {t:10s} ({len(tr):2d}): BM25={b:.0%} Unified={u:.0%} | U={wu} B={wb}")

print(f"\nAvg time: BM25={sum(r['bm25_time'] for r in results)/len(results):.2f}s  Unified={sum(r['unified_time'] for r in results)/len(results):.2f}s")

print(f"\nUnified BETTER:")
for r in results:
    if r['unified_relevance'] > r['bm25_relevance']:
        print(f"  Q{r['id']:3d} [{r['expected_type']:9s}] B={r['bm25_relevance']:.0%} U={r['unified_relevance']:.0%} | {r['question'][:50]}")

print(f"\nUnified WORSE:")
for r in results:
    if r['unified_relevance'] < r['bm25_relevance']:
        print(f"  Q{r['id']:3d} [{r['expected_type']:9s}] B={r['bm25_relevance']:.0%} U={r['unified_relevance']:.0%} | {r['question'][:50]}")

# Save
with open(os.path.join(os.path.dirname(__file__), '..', 'workspace', 'test_finance_180_results.json'), 'w') as f:
    json.dump(results, f, ensure_ascii=False, indent=2)
print(f"\nResults saved.")