app.py 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150
  1. #!/usr/bin/env python3
  2. """
  3. Maritime QA Comparison Demo
  4. RAG vs Wiki side-by-side comparison
  5. With server-side conversation history persistence
  6. """
  7. import json
  8. import os
  9. import sys
  10. import time
  11. import threading
  12. from datetime import datetime
  13. from flask import Flask, request, jsonify, render_template_string
  14. import yaml
  15. # Load paths from agent config
  16. _agent_cfg_path = os.environ.get('AGENT_CONFIG', '/app/agentexample/qaagent67/agent-config.yml')
  17. _wiki_cfg_path = os.environ.get('WIKI_CONFIG', '/app/agentexample/qaagent67wiki/agent-config.yml')
  18. _cfg = {}
  19. _wiki_cfg = {}
  20. if os.path.exists(_agent_cfg_path):
  21. with open(_agent_cfg_path) as f:
  22. _cfg = yaml.safe_load(f)
  23. if os.path.exists(_wiki_cfg_path):
  24. with open(_wiki_cfg_path) as f:
  25. _wiki_cfg = yaml.safe_load(f)
  26. # ═══ instance.yml overlay (domain-specific overrides) ═══
  27. def _deep_merge(base, override):
  28. import copy
  29. result = copy.deepcopy(base)
  30. for k, v in (override or {}).items():
  31. if k in result and isinstance(result[k], dict) and isinstance(v, dict):
  32. result[k] = _deep_merge(result[k], v)
  33. else:
  34. result[k] = copy.deepcopy(v)
  35. return result
  36. _instance_cfg_path = os.environ.get('INSTANCE_CONFIG', '')
  37. if not _instance_cfg_path:
  38. _base = (_cfg.get('knowledge', {}) or {}).get('baseDir', '')
  39. if _base:
  40. _cand = os.path.join(_base, 'instance.yml')
  41. if os.path.isfile(_cand):
  42. _instance_cfg_path = _cand
  43. if _instance_cfg_path and os.path.isfile(_instance_cfg_path):
  44. with open(_instance_cfg_path) as _f:
  45. _instance = yaml.safe_load(_f) or {}
  46. _cfg = _deep_merge(_cfg, _instance)
  47. print('[config] loaded instance overlay:', _instance_cfg_path)
  48. _knowledge = _cfg.get('knowledge', {})
  49. KNOWLEDGE_BASE = _knowledge.get('baseDir', os.environ.get('KNOWLEDGE_DIR', '/data/knowledge/maritime'))
  50. sys.path.insert(0, KNOWLEDGE_BASE)
  51. try:
  52. from search_unified import search
  53. except ImportError:
  54. from search_engine import search
  55. from lambdagent.providers import create_provider
  56. app = Flask(__name__)
  57. HISTORY_FILE = os.environ.get('HISTORY_FILE', '/app/qademo/conversation_history.json')
  58. # WIKI_DIR — prefer instance.yml overlay (_cfg.wiki.dir), then standalone wiki config, then default
  59. WIKI_DIR = (
  60. (_cfg.get('wiki', {}) or {}).get('dir', '')
  61. or (_wiki_cfg.get('wiki', {}) or {}).get('dir', '')
  62. or '/app/agentexample/qaagent67wiki/wiki'
  63. )
  64. # ═══ Demo UI config (domain-specific, loaded from instance.yml) ═══
  65. _demo = _cfg.get('demo', {}) or {}
  66. DEMO_TITLE = _demo.get('title', 'Knowledge QA Demo')
  67. DEMO_TAGLINE = _demo.get('tagline', 'RAG vs Wiki | Qwen2.5:32B local')
  68. DEMO_PLACEHOLDER = _demo.get('placeholder', 'Please enter a question...')
  69. DEMO_DOMAIN = _demo.get('domain', 'knowledge')
  70. DEMO_EXAMPLES = _demo.get('examples', []) # list of {q, label}
  71. DEMO_PORT = int(os.environ.get('PORT') or _demo.get('port', 8080))
  72. DEMO_BIND = os.environ.get('BIND_HOST') or _demo.get('bind') or '0.0.0.0'
  73. # 左右面板默认选项: 海事域左边默认 LambdaRAG,其他域默认 Lite
  74. _is_maritime = 'maritime' in DEMO_DOMAIN.lower() or '海事' in DEMO_TITLE
  75. DEMO_LEFT_DEFAULT = _demo.get('leftDefault', 'rag' if _is_maritime else 'lite')
  76. DEMO_RIGHT_DEFAULT = _demo.get('rightDefault', 'wiki')
  77. def _html_attr_esc(s):
  78. """Escape for HTML attribute in double quotes."""
  79. return (str(s).replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
  80. .replace('"', '&quot;').replace("'", '&#39;'))
  81. def _html_text_esc(s):
  82. return (str(s).replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;'))
  83. _OLD_TITLE_TITLE = '<title>Maritime QA Demo</title>'
  84. _OLD_TITLE_H1 = '<h1>Maritime QA Demo</h1>'
  85. _OLD_TAGLINE = '<p>RAG vs Wiki | 1326 docs | Qwen2.5:32B local</p>'
  86. _OLD_PLACEHOLDER = 'placeholder="Please enter a maritime question..."'
  87. _OLD_EXAMPLES = (
  88. ' <span onclick="askDirect(\'船舶进出港报告制度的主要内容是什么?\')">Ship entry/exit reporting system</span>\n'
  89. ' <span onclick="askDirect(\'船员适任证书的申请条件有哪些?\')">Crew competency certificates</span>\n'
  90. ' <span onclick="askDirect(\'琼州海峡船舶定线制是什么时候开始施行的?\')">Qiongzhou Strait routing system</span>\n'
  91. ' <span onclick="askDirect(\'海上交通安全法与水污染防治法在船舶管理方面的规定有何异同?\')">Maritime law comparison</span>\n'
  92. ' <span onclick="askDirect(\'中国船员发展报告从2019到2023年有什么变化趋势?\')">Crew development trends</span>'
  93. )
  94. def _apply_demo_config(html):
  95. html = html.replace(_OLD_TITLE_TITLE, '<title>' + _html_text_esc(DEMO_TITLE) + '</title>')
  96. html = html.replace(_OLD_TITLE_H1, '<h1>' + _html_text_esc(DEMO_TITLE) + '</h1>')
  97. html = html.replace(_OLD_TAGLINE, '<p>' + _html_text_esc(DEMO_TAGLINE) + '</p>')
  98. html = html.replace(_OLD_PLACEHOLDER, 'placeholder="' + _html_attr_esc(DEMO_PLACEHOLDER) + '"')
  99. if DEMO_EXAMPLES:
  100. lines = []
  101. for e in DEMO_EXAMPLES:
  102. q = str(e.get('q', ''))
  103. label = str(e.get('label', q))
  104. q_esc = _html_attr_esc(q)
  105. label_esc = _html_text_esc(label)
  106. lines.append(' <span onclick="askDirect(\'' + q_esc + '\')">' + label_esc + '</span>')
  107. html = html.replace(_OLD_EXAMPLES, '\n'.join(lines))
  108. # Inject default panel selections as JS
  109. init_script = (
  110. '\n<script>'
  111. 'document.getElementById("leftModeSelect").value = "' + _html_attr_esc(DEMO_LEFT_DEFAULT) + '";'
  112. 'document.getElementById("rightModeSelect").value = "' + _html_attr_esc(DEMO_RIGHT_DEFAULT) + '";'
  113. '</script>\n'
  114. )
  115. html = html.replace('</body>', init_script + '</body>')
  116. return html
  117. # LLM provider (lazy init)
  118. _provider = None
  119. def get_provider():
  120. global _provider
  121. if _provider is None:
  122. _provider = create_provider('ollama', model='qwen2.5-32b', base_url='http://127.0.0.1:8000/v1', timeout=120)
  123. return _provider
  124. # ── Conversation history persistence ──
  125. _history_lock = threading.Lock()
  126. def load_history():
  127. if os.path.exists(HISTORY_FILE):
  128. with open(HISTORY_FILE, 'r', encoding='utf-8') as f:
  129. return json.load(f)
  130. return []
  131. def save_record(record):
  132. with _history_lock:
  133. history = load_history()
  134. history.append(record)
  135. with open(HISTORY_FILE, 'w', encoding='utf-8') as f:
  136. json.dump(history, f, ensure_ascii=False, indent=2)
  137. # ── RAG answer ──
  138. def rag_answer(question, top_k=5):
  139. t0 = time.time()
  140. provider = get_provider()
  141. results = search(question, top_k=top_k)
  142. # Build enriched context from fusion results
  143. enriched = results[0].get('_enriched_context', '') if results else ''
  144. if not enriched:
  145. parts = []
  146. for i, r in enumerate(results):
  147. parts.append('[doc' + str(i+1) + ': ' + r['source'] + ']\n' + r['text'])
  148. enriched = '\n\n'.join(parts)
  149. enriched = enriched[:3500] # fit within 8192 context window
  150. _sys = _cfg.get('systemPrompt', '').strip()
  151. _sys_short = _sys[:500] if _sys else 'You are a ' + DEMO_DOMAIN + ' knowledge QA assistant.'
  152. prompt = f"""{_sys_short}
  153. 请严格基于以下参考资料回答问题。资料来自多个来源(检索文档、知识图谱、Wiki知识),请综合利用。
  154. 如果资料中没有相关信息,请说明"资料中未找到相关信息"。
  155. 回答要具体,引用原文,标注来源 [来源: 文件名]。
  156. {enriched}
  157. ## 问题
  158. {question}
  159. ## 回答"""
  160. answer = provider.chat([{'role': 'user', 'content': prompt}])
  161. elapsed = time.time() - t0
  162. # Extract unified search metadata
  163. query_type = results[0].get('_query_type', 'unknown') if results else 'unknown'
  164. strategy = results[0].get('_strategy', 'unknown') if results else 'unknown'
  165. search_time = results[0].get('_total_time', 0) if results else 0
  166. return {
  167. 'answer': answer,
  168. 'sources': [r['source'] for r in results],
  169. 'elapsed': round(elapsed, 1),
  170. 'chunks_used': len(results),
  171. 'mode': 'LambdaRAG',
  172. 'query_type': query_type,
  173. 'strategy': strategy,
  174. 'search_time': search_time,
  175. }
  176. # ── Wiki answer ──
  177. def wiki_answer(question, top_k=5):
  178. t0 = time.time()
  179. provider = get_provider()
  180. from pathlib import Path
  181. wiki_path = Path(WIKI_DIR)
  182. wiki_pages = []
  183. for subdir in ['sources', 'entities', 'topics', 'analyses']:
  184. d = wiki_path / subdir
  185. if d.exists():
  186. for f in d.glob('*.md'):
  187. try:
  188. content = f.read_text(encoding='utf-8')
  189. q_chars = set(question)
  190. match_score = sum(1 for c in q_chars if c in content)
  191. if match_score > len(question) * 0.3:
  192. wiki_pages.append({
  193. 'name': f.stem,
  194. 'content': content[:1500],
  195. 'score': match_score,
  196. })
  197. except:
  198. pass
  199. wiki_pages.sort(key=lambda x: x['score'], reverse=True)
  200. wiki_pages = wiki_pages[:3]
  201. if wiki_pages:
  202. wiki_context = '\n\n'.join([
  203. f'[Wiki: {p["name"]}]\n{p["content"]}' for p in wiki_pages
  204. ])
  205. source_type = 'wiki'
  206. else:
  207. results = search(question, top_k=top_k)
  208. compile_context = '\n\n'.join([
  209. f'[doc: {r["source"]}]\n{r["text"]}' for r in results
  210. ])
  211. compile_prompt = f"""请阅读以下文档片段,提炼出与问题相关的核心知识点。
  212. 用结构化的方式组织,标注来源。
  213. 文档:
  214. {compile_context}
  215. 问题: {question}
  216. 请输出结构化的知识摘要:"""
  217. compiled = provider.chat([{'role': 'user', 'content': compile_prompt}])
  218. wiki_context = compiled
  219. source_type = 'compiled'
  220. safe_name = question[:30].replace('/', '_').replace(' ', '_')
  221. analysis_path = wiki_path / 'analyses' / f'{safe_name}.md'
  222. analysis_path.parent.mkdir(parents=True, exist_ok=True)
  223. with open(analysis_path, 'w', encoding='utf-8') as f:
  224. f.write(f'# {question}\n\n{compiled}\n')
  225. answer_prompt = f"""You are a {DEMO_DOMAIN} wiki knowledge QA assistant.
  226. 基于以下已编译的 wiki 知识回答问题。回答要具体,标注来源。
  227. ## Wiki 知识
  228. {wiki_context}
  229. ## 问题
  230. {question}
  231. ## 回答"""
  232. answer = provider.chat([{'role': 'user', 'content': answer_prompt}])
  233. elapsed = time.time() - t0
  234. return {
  235. 'answer': answer,
  236. 'source_type': source_type,
  237. 'wiki_pages_used': len(wiki_pages),
  238. 'elapsed': round(elapsed, 1),
  239. 'mode': 'Wiki',
  240. }
  241. # ── Async results store ──
  242. # -- BM25 Baseline answer (for comparison) --
  243. def bm25_baseline_answer(question, top_k=5):
  244. t0 = time.time()
  245. provider = get_provider()
  246. try:
  247. from search_engine import search as bm25_only
  248. results = bm25_only(question, top_k=top_k)
  249. except Exception:
  250. results = []
  251. context_parts = []
  252. for i, r in enumerate(results):
  253. context_parts.append('[doc' + str(i+1) + ': ' + r['source'] + ']' + chr(10) + r['text'])
  254. context = (chr(10)*2).join(context_parts)
  255. prompt = ('You are a ' + DEMO_DOMAIN + ' QA assistant. Answer based on docs only.' + chr(10)
  256. + '## Docs' + chr(10) + context + chr(10)
  257. + '## Question' + chr(10) + question + chr(10) + '## Answer')
  258. answer = provider.chat([{'role': 'user', 'content': prompt}])
  259. elapsed = time.time() - t0
  260. return {
  261. 'answer': answer,
  262. 'sources': [r['source'] for r in results],
  263. 'elapsed': round(elapsed, 1),
  264. 'chunks_used': len(results),
  265. 'mode': 'BM25 Baseline',
  266. }
  267. # -- Direct LLM answer (no retrieval, pure parametric knowledge) --
  268. def direct_llm_answer(question):
  269. t0 = time.time()
  270. provider = get_provider()
  271. prompt = ('You are a ' + DEMO_DOMAIN + ' QA assistant. '
  272. 'Answer the following question using ONLY your own knowledge. '
  273. 'If you are not sure, say so.\n\n'
  274. '## Question\n' + question + '\n\n## Answer')
  275. answer = provider.chat([{'role': 'user', 'content': prompt}])
  276. elapsed = time.time() - t0
  277. return {
  278. 'answer': answer,
  279. 'elapsed': round(elapsed, 1),
  280. 'mode': 'Direct LLM',
  281. 'chunks_used': 0,
  282. 'sources': [],
  283. }
  284. # ── Lite answer (BM25 + Wiki combined — qaagent67lite strategy) ──
  285. def lite_answer(question, top_k=5):
  286. """qaagent67lite: BM25 关键词检索 + Wiki 编译知识融合"""
  287. t0 = time.time()
  288. provider = get_provider()
  289. from pathlib import Path
  290. # Path 1: BM25
  291. bm25_results = []
  292. try:
  293. from search_engine import search as bm25_search
  294. bm25_results = bm25_search(question, top_k=top_k)
  295. except Exception:
  296. pass
  297. bm25_context = '\n\n'.join([
  298. f'[BM25 doc: {r["source"]}]\n{r["text"]}' for r in bm25_results[:5]
  299. ]) if bm25_results else ''
  300. # Path 2: Wiki
  301. wiki_path = Path(WIKI_DIR)
  302. wiki_pages = []
  303. for subdir in ['entities', 'topics', 'sources', 'analyses']:
  304. d = wiki_path / subdir
  305. if d.exists():
  306. for f in d.glob('*.md'):
  307. try:
  308. content = f.read_text(encoding='utf-8')
  309. q_chars = set(question)
  310. match_score = sum(1 for c in q_chars if c in content)
  311. if match_score > len(question) * 0.3:
  312. wiki_pages.append({
  313. 'name': f.stem,
  314. 'subdir': subdir,
  315. 'content': content[:1500],
  316. 'score': match_score,
  317. })
  318. except Exception:
  319. pass
  320. wiki_pages.sort(key=lambda x: x['score'], reverse=True)
  321. wiki_pages = wiki_pages[:3]
  322. wiki_context = '\n\n'.join([
  323. f'[Wiki/{p["subdir"]}: {p["name"]}]\n{p["content"]}' for p in wiki_pages
  324. ]) if wiki_pages else ''
  325. # Merge contexts
  326. merged = ''
  327. if wiki_context:
  328. merged += '## Wiki 编译知识\n' + wiki_context + '\n\n'
  329. if bm25_context:
  330. merged += '## BM25 检索片段\n' + bm25_context
  331. if not merged.strip():
  332. return {
  333. 'answer': '知识库中未找到相关信息。',
  334. 'elapsed': round(time.time() - t0, 1),
  335. 'mode': 'Lite (BM25+Wiki)',
  336. 'bm25_chunks': 0,
  337. 'wiki_pages_used': 0,
  338. }
  339. _sys = _cfg.get('systemPrompt', '').strip()
  340. _sys_short = _sys[:500] if _sys else 'You are a ' + DEMO_DOMAIN + ' knowledge QA assistant.'
  341. prompt = f"""{_sys_short}
  342. 请基于以下两种来源回答问题。Wiki 编译知识是经过整理的结构化知识(优先参考),BM25 检索片段是原始文档片段(用于补充细节)。
  343. 如果两种来源有矛盾,以 Wiki 知识为准。标注信息来源。
  344. {merged}
  345. ## 问题
  346. {question}
  347. ## 回答"""
  348. answer = provider.chat([{'role': 'user', 'content': prompt}])
  349. elapsed = time.time() - t0
  350. return {
  351. 'answer': answer,
  352. 'elapsed': round(elapsed, 1),
  353. 'mode': 'Lite (BM25+Wiki)',
  354. 'bm25_chunks': len(bm25_results),
  355. 'wiki_pages_used': len(wiki_pages),
  356. 'sources': [r['source'] for r in bm25_results[:3]] + [p['name'] for p in wiki_pages],
  357. }
  358. _results = {}
  359. # ── API routes ──
  360. @app.route('/api/ask', methods=['POST'])
  361. def api_ask():
  362. data = request.json
  363. question = data.get('question', '').strip()
  364. if not question:
  365. return jsonify({'error': 'question is required'}), 400
  366. req_id = str(int(time.time() * 1000))
  367. _results[req_id] = {'lite': None, 'rag': None, 'wiki': None, 'question': question}
  368. def run_lite():
  369. try:
  370. _results[req_id]['lite'] = lite_answer(question)
  371. except Exception as e:
  372. _results[req_id]['lite'] = {'answer': f'Error: {e}', 'elapsed': 0, 'mode': 'Lite'}
  373. def run_rag():
  374. try:
  375. _results[req_id]['rag'] = rag_answer(question)
  376. except Exception as e:
  377. _results[req_id]['rag'] = {'answer': f'Error: {e}', 'elapsed': 0, 'mode': 'RAG'}
  378. def run_wiki():
  379. try:
  380. _results[req_id]['wiki'] = wiki_answer(question)
  381. except Exception as e:
  382. _results[req_id]['wiki'] = {'answer': f'Error: {e}', 'elapsed': 0, 'mode': 'Wiki'}
  383. def run_baseline():
  384. try:
  385. _results[req_id]['baseline'] = bm25_baseline_answer(question)
  386. except Exception as e:
  387. _results[req_id]['baseline'] = {'answer': 'Error: ' + str(e), 'elapsed': 0, 'mode': 'BM25 Baseline'}
  388. def run_direct():
  389. try:
  390. _results[req_id]['direct'] = direct_llm_answer(question)
  391. except Exception as e:
  392. _results[req_id]['direct'] = {'answer': 'Error: ' + str(e), 'elapsed': 0, 'mode': 'Direct LLM'}
  393. t0 = threading.Thread(target=run_lite)
  394. t1 = threading.Thread(target=run_rag)
  395. t2 = threading.Thread(target=run_wiki)
  396. t3 = threading.Thread(target=run_baseline)
  397. t4 = threading.Thread(target=run_direct)
  398. t0.start()
  399. t1.start()
  400. t2.start()
  401. t3.start()
  402. t4.start()
  403. return jsonify({'req_id': req_id, 'status': 'processing'})
  404. @app.route('/api/result/<req_id>')
  405. def api_result(req_id):
  406. if req_id not in _results:
  407. return jsonify({'error': 'not found'}), 404
  408. r = _results[req_id]
  409. # Done when all 5 modes have results
  410. done = all(r.get(k) is not None for k in ('lite', 'rag', 'wiki', 'baseline', 'direct'))
  411. # Save to history when all done
  412. if done and not r.get('_saved'):
  413. r['_saved'] = True
  414. record = {
  415. 'id': req_id,
  416. 'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
  417. 'question': r['question'],
  418. 'lite': r['lite'],
  419. 'rag': r['rag'],
  420. 'wiki': r.get('wiki'),
  421. 'baseline': r.get('baseline'),
  422. 'direct': r.get('direct'),
  423. }
  424. threading.Thread(target=save_record, args=(record,)).start()
  425. return jsonify({
  426. 'status': 'done' if done else 'processing',
  427. 'lite': r.get('lite'),
  428. 'rag': r.get('rag'),
  429. 'wiki': r.get('wiki'),
  430. 'baseline': r.get('baseline'),
  431. 'direct': r.get('direct'),
  432. })
  433. @app.route('/api/history')
  434. def api_history():
  435. history = load_history()
  436. history.reverse() # newest first
  437. return jsonify(history[:50]) # last 50
  438. @app.route('/api/history/export')
  439. def api_history_export():
  440. history = load_history()
  441. return jsonify({
  442. 'exported_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
  443. 'total_records': len(history),
  444. 'records': history,
  445. })
  446. @app.route('/api/wiki/index')
  447. def api_wiki_index():
  448. """List all wiki pages grouped by category"""
  449. from pathlib import Path
  450. wiki_path = Path(WIKI_DIR)
  451. result = {}
  452. for subdir in ['sources', 'entities', 'topics', 'analyses']:
  453. d = wiki_path / subdir
  454. pages = []
  455. if d.exists():
  456. for f in sorted(d.glob('*.md')):
  457. stat = f.stat()
  458. pages.append({
  459. 'name': f.stem,
  460. 'path': f'{subdir}/{f.name}',
  461. 'size': stat.st_size,
  462. 'modified': datetime.fromtimestamp(stat.st_mtime).strftime('%Y-%m-%d %H:%M'),
  463. })
  464. result[subdir] = pages
  465. # Also read index.md
  466. idx_path = wiki_path / 'index.md'
  467. result['index_content'] = idx_path.read_text(encoding='utf-8') if idx_path.exists() else ''
  468. return jsonify(result)
  469. @app.route('/api/wiki/page/<path:page_path>')
  470. def api_wiki_page(page_path):
  471. """Read a single wiki page"""
  472. from pathlib import Path
  473. wiki_path = Path(WIKI_DIR)
  474. full_path = wiki_path / page_path
  475. if not full_path.exists() or not str(full_path).startswith(str(wiki_path)):
  476. return jsonify({'error': 'not found'}), 404
  477. content = full_path.read_text(encoding='utf-8')
  478. return jsonify({'path': page_path, 'content': content, 'size': len(content)})
  479. @app.route('/api/rag/search')
  480. def api_rag_search():
  481. """Search RAG chunks"""
  482. query = request.args.get('q', '').strip()
  483. top_k = int(request.args.get('k', 10))
  484. if not query:
  485. return jsonify({'error': 'q parameter required'}), 400
  486. results = search(query, top_k=top_k)
  487. return jsonify({'query': query, 'total_results': len(results), 'results': results})
  488. @app.route('/api/rag/stats')
  489. def api_rag_stats():
  490. """RAG index statistics"""
  491. import json as json_mod
  492. from pathlib import Path
  493. idx_path = Path(_knowledge.get('indexFile', KNOWLEDGE_BASE + '/rag_index.json'))
  494. if not idx_path.exists():
  495. return jsonify({'error': 'index not built'}), 404
  496. with open(idx_path) as f:
  497. data = json_mod.load(f)
  498. # Sample chunks for preview
  499. sample_chunks = data['chunks'][:20]
  500. for c in sample_chunks:
  501. pass # show full text
  502. return jsonify({
  503. 'total_chunks': data['total_chunks'],
  504. 'total_files': data['total_files'],
  505. 'chunk_size': data['chunk_size'],
  506. 'keyword_index_size': data.get('keyword_index_size', 0),
  507. 'sample_chunks': sample_chunks,
  508. })
  509. @app.route('/wiki')
  510. def wiki_page():
  511. return render_template_string(WIKI_HTML)
  512. @app.route('/rag')
  513. def rag_page():
  514. return render_template_string(RAG_HTML)
  515. @app.route('/')
  516. def index():
  517. return _apply_demo_config(render_template_string(HTML_TEMPLATE))
  518. HTML_TEMPLATE = r"""
  519. <!DOCTYPE html>
  520. <html lang="zh-CN">
  521. <head>
  522. <meta charset="UTF-8">
  523. <meta name="viewport" content="width=device-width, initial-scale=1.0">
  524. <title>Maritime QA Demo</title>
  525. <style>
  526. * { margin: 0; padding: 0; box-sizing: border-box; }
  527. body { font-family: -apple-system, "PingFang SC", "Microsoft YaHei", sans-serif; background: #f0f2f5; color: #333; }
  528. .header { background: linear-gradient(135deg, #1a3a5c 0%, #2d6aa0 100%); color: white; padding: 20px 30px; }
  529. .header-inner { max-width: 1400px; margin: 0 auto; display: flex; justify-content: space-between; align-items: center; }
  530. .header h1 { font-size: 22px; }
  531. .header p { opacity: 0.8; font-size: 13px; margin-top: 4px; }
  532. .header-stats { text-align: right; font-size: 13px; opacity: 0.8; }
  533. .container { max-width: 1400px; margin: 0 auto; padding: 20px; }
  534. .input-area { background: white; border-radius: 12px; padding: 20px; margin-bottom: 20px; box-shadow: 0 2px 8px rgba(0,0,0,0.08); }
  535. .input-row { display: flex; gap: 12px; }
  536. .input-row input { flex: 1; padding: 12px 16px; border: 2px solid #e0e0e0; border-radius: 8px; font-size: 16px; outline: none; transition: border-color 0.2s; }
  537. .input-row input:focus { border-color: #2d6aa0; }
  538. .btn { padding: 12px 28px; background: #2d6aa0; color: white; border: none; border-radius: 8px; font-size: 16px; cursor: pointer; transition: background 0.2s; white-space: nowrap; }
  539. .btn:hover { background: #1a5a8e; }
  540. .btn:disabled { background: #aaa; cursor: not-allowed; }
  541. .btn-sm { padding: 6px 14px; font-size: 13px; border-radius: 6px; }
  542. .btn-outline { background: transparent; border: 1px solid #2d6aa0; color: #2d6aa0; }
  543. .btn-outline:hover { background: #e0eef8; }
  544. .quick-questions { margin-top: 12px; display: flex; flex-wrap: wrap; gap: 8px; }
  545. .quick-questions span { padding: 6px 12px; background: #f5f5f5; border-radius: 16px; font-size: 13px; cursor: pointer; transition: background 0.2s; border: 1px solid transparent; }
  546. .quick-questions span:hover { background: #e0eef8; border-color: #c0d8f0; }
  547. .compare-area { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
  548. .panel { background: white; border-radius: 12px; padding: 20px; box-shadow: 0 2px 8px rgba(0,0,0,0.08); min-height: 200px; }
  549. .panel-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 16px; padding-bottom: 12px; border-bottom: 2px solid #f0f0f0; }
  550. .panel-title { font-size: 18px; font-weight: 600; }
  551. .panel-title.rag { color: #2d8a5e; }
  552. .panel-title.wiki { color: #7b2d8e; }
  553. .badge-lite { background: #e0f0e8; color: #2d8a5e; }
  554. .panel-badge { padding: 4px 10px; border-radius: 12px; font-size: 12px; font-weight: 500; }
  555. .badge-rag { background: #e0eef8; color: #2d6aa0; }
  556. .badge-wiki { background: #f0e0f8; color: #7b2d8e; }
  557. .panel-content { font-size: 15px; line-height: 1.8; white-space: pre-wrap; word-wrap: break-word; max-height: 600px; overflow-y: auto; }
  558. .panel-meta { margin-top: 12px; padding-top: 12px; border-top: 1px solid #f0f0f0; font-size: 13px; color: #888; }
  559. .loading { text-align: center; padding: 40px; color: #888; }
  560. .spinner { display: inline-block; width: 20px; height: 20px; border: 3px solid #e0e0e0; border-top-color: #2d6aa0; border-radius: 50%; animation: spin 0.8s linear infinite; margin-right: 8px; vertical-align: middle; }
  561. @keyframes spin { to { transform: rotate(360deg); } }
  562. .tabs { display: flex; gap: 0; margin-top: 20px; margin-bottom: 0; }
  563. .tab { padding: 10px 20px; background: #e8e8e8; cursor: pointer; font-size: 14px; border-radius: 8px 8px 0 0; }
  564. .tab.active { background: white; font-weight: 600; }
  565. .tab-content { background: white; border-radius: 0 12px 12px 12px; padding: 16px 20px; box-shadow: 0 2px 8px rgba(0,0,0,0.08); display: none; max-height: 400px; overflow-y: auto; }
  566. .tab-content.active { display: block; }
  567. .hist-item { padding: 12px 0; border-bottom: 1px solid #f5f5f5; cursor: pointer; }
  568. .hist-item:hover { background: #fafafa; }
  569. .hist-item:last-child { border-bottom: none; }
  570. .hist-q { font-weight: 600; font-size: 14px; margin-bottom: 4px; }
  571. .hist-meta { font-size: 12px; color: #999; }
  572. .empty-hint { text-align: center; color: #ccc; padding: 30px; }
  573. .mode-select { padding: 4px 10px; border-radius: 8px; border: 1px solid #7b2d8e; background: #f0e0f8; color: #7b2d8e; font-size: 13px; font-weight: 500; cursor: pointer; outline: none; }
  574. .mode-select:focus { border-color: #5a1a6e; }
  575. .mode-select-left { border-color: #2d6aa0; background: #e0eef8; color: #2d6aa0; }
  576. .mode-select-left:focus { border-color: #1a5a8e; }
  577. .badge-left { background: #e0eef8; color: #2d6aa0; }
  578. @media (max-width: 768px) { .compare-area { grid-template-columns: 1fr; } }
  579. </style>
  580. </head>
  581. <body>
  582. <div class="header">
  583. <div class="header-inner">
  584. <div>
  585. <h1>Maritime QA Demo</h1>
  586. <p>RAG vs Wiki | 1326 docs | Qwen2.5:32B local</p>
  587. </div>
  588. <div class="header-stats">
  589. <div style="margin-bottom:6px;">
  590. <a href="/" style="color:white;text-decoration:none;margin-right:16px;font-weight:600;">QA Demo</a>
  591. <a href="/wiki" style="color:white;text-decoration:none;margin-right:16px;opacity:0.85;">Wiki Browser</a>
  592. <a href="/rag" style="color:white;text-decoration:none;opacity:0.85;">RAG Chunks</a>
  593. </div>
  594. <div id="statsLine">-</div>
  595. </div>
  596. </div>
  597. </div>
  598. <div class="container">
  599. <div class="input-area">
  600. <div class="input-row">
  601. <input type="text" id="question" placeholder="Please enter a maritime question..." autofocus>
  602. <button class="btn" id="askBtn" onclick="askQuestion()">Ask</button>
  603. </div>
  604. <div class="quick-questions">
  605. <span onclick="askDirect('船舶进出港报告制度的主要内容是什么?')">Ship entry/exit reporting system</span>
  606. <span onclick="askDirect('船员适任证书的申请条件有哪些?')">Crew competency certificates</span>
  607. <span onclick="askDirect('琼州海峡船舶定线制是什么时候开始施行的?')">Qiongzhou Strait routing system</span>
  608. <span onclick="askDirect('海上交通安全法与水污染防治法在船舶管理方面的规定有何异同?')">Maritime law comparison</span>
  609. <span onclick="askDirect('中国船员发展报告从2019到2023年有什么变化趋势?')">Crew development trends</span>
  610. </div>
  611. </div>
  612. <div class="compare-area">
  613. <div class="panel" id="leftPanel">
  614. <div class="panel-header">
  615. <select id="leftModeSelect" class="mode-select mode-select-left" onchange="onPanelModeChange('left')">
  616. <option value="lite">Lite (BM25 + Wiki)</option>
  617. <option value="rag">LambdaRAG (4-path)</option>
  618. <option value="wiki">Wiki Only</option>
  619. <option value="baseline">BM25 Baseline</option>
  620. <option value="direct">Direct LLM</option>
  621. </select>
  622. <span class="panel-badge badge-left" id="leftBadge">-</span>
  623. </div>
  624. <div class="panel-content" id="leftContent" style="white-space:pre-wrap;">
  625. <div class="empty-hint">Ask to see response</div>
  626. </div>
  627. <div class="panel-meta" id="leftMeta"></div>
  628. </div>
  629. <div class="panel" id="rightPanel">
  630. <div class="panel-header">
  631. <select id="rightModeSelect" class="mode-select" onchange="onPanelModeChange('right')">
  632. <option value="rag">LambdaRAG (4-path)</option>
  633. <option value="lite">Lite (BM25 + Wiki)</option>
  634. <option value="wiki">Wiki Only</option>
  635. <option value="baseline">BM25 Baseline</option>
  636. <option value="direct">Direct LLM</option>
  637. </select>
  638. <span class="panel-badge badge-wiki" id="rightBadge">-</span>
  639. </div>
  640. <div class="panel-content" id="rightContent" style="white-space:pre-wrap;">
  641. <div class="empty-hint">Ask to see comparison</div>
  642. </div>
  643. <div class="panel-meta" id="rightMeta"></div>
  644. </div>
  645. </div>
  646. <div class="tabs">
  647. <div class="tab active" onclick="switchTab('history')">History</div>
  648. <div class="tab" onclick="switchTab('export')">Export</div>
  649. </div>
  650. <div class="tab-content active" id="tab-history">
  651. <div class="empty-hint" id="historyEmpty">No history yet</div>
  652. <div id="historyList"></div>
  653. </div>
  654. <div class="tab-content" id="tab-export">
  655. <p style="margin-bottom:12px;color:#666;">All records are saved on the server. Click to export.</p>
  656. <button class="btn btn-sm btn-outline" onclick="exportHistory()">Export JSON</button>
  657. <pre id="exportPreview" style="margin-top:12px;font-size:12px;max-height:300px;overflow:auto;background:#f8f8f8;padding:12px;border-radius:8px;display:none;"></pre>
  658. </div>
  659. </div>
  660. <script>
  661. const qInput = document.getElementById('question');
  662. const askBtn = document.getElementById('askBtn');
  663. qInput.addEventListener('keydown', e => { if (e.key === 'Enter') askQuestion(); });
  664. function askDirect(q) { qInput.value = q; askQuestion(); }
  665. function askQuestion() {
  666. const q = qInput.value.trim();
  667. if (!q) return;
  668. askBtn.disabled = true;
  669. askBtn.textContent = 'Processing...';
  670. document.getElementById('leftContent').innerHTML = '<div class="loading"><span class="spinner"></span>Loading...</div>';
  671. document.getElementById('rightContent').innerHTML = '<div class="loading"><span class="spinner"></span>Loading...</div>';
  672. document.getElementById('leftMeta').textContent = '';
  673. document.getElementById('rightMeta').textContent = '';
  674. fetch('/api/ask', {
  675. method: 'POST',
  676. headers: {'Content-Type': 'application/json'},
  677. body: JSON.stringify({question: q})
  678. }).then(r => r.json()).then(data => pollResult(data.req_id, q));
  679. }
  680. var _currentResult = null;
  681. function displayPanel(side, data) {
  682. var selectId = side + 'ModeSelect';
  683. var contentId = side + 'Content';
  684. var badgeId = side + 'Badge';
  685. var metaId = side + 'Meta';
  686. var mode = document.getElementById(selectId).value;
  687. var src = data[mode] || null;
  688. if (!src) {
  689. document.getElementById(contentId).innerHTML = '<div class="loading"><span class="spinner"></span>Loading ' + mode + '...</div>';
  690. document.getElementById(metaId).textContent = '';
  691. return;
  692. }
  693. document.getElementById(contentId).textContent = src.answer;
  694. var srcs = src.sources ? src.sources.slice(0,3).join(', ') : '';
  695. if (mode === 'rag') {
  696. document.getElementById(badgeId).textContent = (src.query_type||'') + ' → ' + (src.strategy||'Unified');
  697. document.getElementById(metaId).innerHTML =
  698. '<b>Time:</b> ' + src.elapsed + 's | <b>Strategy:</b> ' + (src.strategy||'-') +
  699. ' | <b>Chunks:</b> ' + (src.chunks_used||0) + '<br><b>Sources:</b> ' + srcs;
  700. } else if (mode === 'lite') {
  701. document.getElementById(badgeId).textContent =
  702. 'BM25:' + (src.bm25_chunks||0) + ' + Wiki:' + (src.wiki_pages_used||0);
  703. document.getElementById(metaId).innerHTML =
  704. '<b>Time:</b> ' + src.elapsed + 's | <b>BM25:</b> ' + (src.bm25_chunks||0) +
  705. ' | <b>Wiki:</b> ' + (src.wiki_pages_used||0) + '<br><b>Sources:</b> ' + srcs;
  706. } else if (mode === 'wiki') {
  707. document.getElementById(badgeId).textContent = 'Compile + Query';
  708. document.getElementById(metaId).innerHTML =
  709. '<b>Time:</b> ' + src.elapsed + 's | <b>Wiki pages:</b> ' + (src.wiki_pages_used||0);
  710. } else if (mode === 'baseline') {
  711. document.getElementById(badgeId).textContent = 'BM25 Keyword';
  712. document.getElementById(metaId).innerHTML =
  713. '<b>Time:</b> ' + src.elapsed + 's | <b>Chunks:</b> ' + (src.chunks_used||0) +
  714. ' | <b>Sources:</b> ' + srcs;
  715. } else if (mode === 'direct') {
  716. document.getElementById(badgeId).textContent = 'No Retrieval';
  717. document.getElementById(metaId).innerHTML =
  718. '<b>Time:</b> ' + src.elapsed + 's | <b>Mode:</b> Pure LLM';
  719. }
  720. }
  721. function onPanelModeChange(side) {
  722. if (_currentResult) displayPanel(side, _currentResult);
  723. }
  724. function pollResult(reqId, question) {
  725. const poll = setInterval(() => {
  726. fetch('/api/result/' + reqId).then(r => r.json()).then(data => {
  727. _currentResult = data;
  728. displayPanel('left', data);
  729. displayPanel('right', data);
  730. if (data.status === 'done') {
  731. clearInterval(poll);
  732. askBtn.disabled = false;
  733. askBtn.textContent = 'Ask';
  734. loadHistory();
  735. }
  736. });
  737. }, 2000);
  738. }
  739. function loadHistory() {
  740. fetch('/api/history').then(r => r.json()).then(data => {
  741. const el = document.getElementById('historyList');
  742. const empty = document.getElementById('historyEmpty');
  743. if (!data.length) { empty.style.display = 'block'; el.innerHTML = ''; return; }
  744. empty.style.display = 'none';
  745. document.getElementById('statsLine').textContent = 'Total records: ' + data.length;
  746. el.innerHTML = data.map((h, i) =>
  747. '<div class="hist-item" onclick="showRecord(' + i + ')">' +
  748. '<div class="hist-q">' + escHtml(h.question) + '</div>' +
  749. '<div class="hist-meta">' + h.timestamp +
  750. ' | Lite: ' + (h.lite?h.lite.elapsed:'-') + 's' +
  751. ' | RAG: ' + (h.rag?h.rag.elapsed:'-') + 's</div></div>'
  752. ).join('');
  753. window._histData = data;
  754. });
  755. }
  756. function showRecord(idx) {
  757. const h = window._histData[idx];
  758. if (!h) return;
  759. qInput.value = h.question;
  760. _currentResult = h;
  761. displayPanel('left', h);
  762. displayPanel('right', h);
  763. window.scrollTo({top: 0, behavior: 'smooth'});
  764. }
  765. function switchTab(name) {
  766. document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
  767. document.querySelectorAll('.tab-content').forEach(t => t.classList.remove('active'));
  768. event.target.classList.add('active');
  769. document.getElementById('tab-' + name).classList.add('active');
  770. }
  771. function exportHistory() {
  772. fetch('/api/history/export').then(r => r.json()).then(data => {
  773. const pre = document.getElementById('exportPreview');
  774. pre.style.display = 'block';
  775. pre.textContent = JSON.stringify(data, null, 2);
  776. });
  777. }
  778. function escHtml(s) {
  779. const d = document.createElement('div');
  780. d.textContent = s;
  781. return d.innerHTML;
  782. }
  783. function renderAnswer(text) {
  784. if (typeof marked === "undefined") return escHtml(text);
  785. var clean = text.replace(/([^\n])\n([^\n\-\#\*\|])/g, "$1\n\n$2");
  786. clean = clean.replace(/([^\n])\n(- )/g, "$1\n\n$2");
  787. clean = clean.replace(/([^\n])\n(#{1,3} )/g, "$1\n\n$2");
  788. try { return marked.parse(clean); } catch(e) { return "<pre>" + escHtml(text) + "</pre>"; }
  789. }
  790. // Load history on page load
  791. loadHistory();
  792. </script>
  793. </body>
  794. </html>
  795. """
  796. WIKI_HTML = r"""
  797. <!DOCTYPE html>
  798. <html lang="zh-CN">
  799. <head>
  800. <meta charset="UTF-8">
  801. <meta name="viewport" content="width=device-width, initial-scale=1.0">
  802. <title>Wiki Browser</title>
  803. <style>
  804. * { margin:0; padding:0; box-sizing:border-box; }
  805. body { font-family: -apple-system,"PingFang SC","Microsoft YaHei",sans-serif; background:#f0f2f5; color:#333; }
  806. .header { background:linear-gradient(135deg,#5b2d8e 0%,#7b4dae 100%); color:white; padding:16px 30px; }
  807. .header-inner { max-width:1400px; margin:0 auto; display:flex; justify-content:space-between; align-items:center; }
  808. .header h1 { font-size:22px; }
  809. .header a { color:white; text-decoration:none; opacity:0.8; }
  810. .header a:hover { opacity:1; }
  811. .container { max-width:1400px; margin:0 auto; padding:20px; display:flex; gap:20px; }
  812. .sidebar { width:300px; flex-shrink:0; }
  813. .main { flex:1; min-width:0; }
  814. .card { background:white; border-radius:12px; padding:16px; margin-bottom:16px; box-shadow:0 2px 8px rgba(0,0,0,0.08); }
  815. .card h3 { font-size:15px; margin-bottom:10px; color:#5b2d8e; }
  816. .page-list { max-height:250px; overflow-y:auto; }
  817. .page-item { padding:6px 8px; cursor:pointer; font-size:13px; border-radius:6px; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
  818. .page-item:hover { background:#f0e0f8; }
  819. .page-item.active { background:#e0d0f0; font-weight:600; }
  820. .badge { display:inline-block; padding:2px 8px; border-radius:10px; font-size:11px; background:#f0e0f8; color:#5b2d8e; margin-left:6px; }
  821. .content-area { font-size:15px; line-height:1.8; word-wrap:break-word; max-height:700px; overflow-y:auto; }
  822. .content-area h1 { font-size:20px; margin:16px 0 8px; color:#333; border-bottom:1px solid #eee; padding-bottom:6px; }
  823. .content-area h2 { font-size:17px; margin:14px 0 6px; color:#444; }
  824. .content-area h3 { font-size:15px; margin:10px 0 4px; color:#555; }
  825. .content-area ul, .content-area ol { padding-left:20px; margin:8px 0; }
  826. .content-area li { margin:4px 0; }
  827. .content-area p { margin:8px 0; }
  828. .content-area code { background:#f0f0f0; padding:1px 4px; border-radius:3px; font-size:13px; }
  829. .content-area blockquote { border-left:3px solid #ccc; padding-left:12px; color:#666; margin:8px 0; }
  830. .stats-row { display:flex; gap:12px; margin-bottom:16px; }
  831. .stat-box { flex:1; background:white; border-radius:10px; padding:14px; text-align:center; box-shadow:0 1px 4px rgba(0,0,0,0.06); }
  832. .stat-num { font-size:28px; font-weight:700; color:#5b2d8e; }
  833. .stat-label { font-size:12px; color:#888; margin-top:4px; }
  834. .search-box { width:100%; padding:8px 12px; border:1px solid #ddd; border-radius:6px; font-size:14px; margin-bottom:10px; }
  835. </style>
  836. </head>
  837. <body>
  838. <div class="header">
  839. <div class="header-inner">
  840. <h1>Wiki Browser</h1>
  841. <div><a href="/">QA Demo</a> | <a href="/rag">RAG Chunks</a></div>
  842. </div>
  843. </div>
  844. <div class="container">
  845. <div class="sidebar">
  846. <div class="stats-row">
  847. <div class="stat-box"><div class="stat-num" id="nSources">-</div><div class="stat-label">Sources</div></div>
  848. <div class="stat-box"><div class="stat-num" id="nEntities">-</div><div class="stat-label">Entities</div></div>
  849. <div class="stat-box"><div class="stat-num" id="nTopics">-</div><div class="stat-label">Topics</div></div>
  850. </div>
  851. <div class="card"><h3>Sources</h3><input class="search-box" placeholder="Filter..." oninput="filterList('sources',this.value)"><div class="page-list" id="list-sources"></div></div>
  852. <div class="card"><h3>Entities</h3><input class="search-box" placeholder="Filter..." oninput="filterList('entities',this.value)"><div class="page-list" id="list-entities"></div></div>
  853. <div class="card"><h3>Topics</h3><div class="page-list" id="list-topics"></div></div>
  854. <div class="card"><h3>Analyses</h3><div class="page-list" id="list-analyses"></div></div>
  855. </div>
  856. <div class="main">
  857. <div class="card">
  858. <h3 id="pageTitle">Wiki Index</h3>
  859. <div class="content-area" id="pageContent">Loading...</div>
  860. <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
  861. </div>
  862. </div>
  863. </div>
  864. <script>
  865. let wikiData = {};
  866. function loadWiki() {
  867. fetch('/api/wiki/index').then(r=>r.json()).then(data => {
  868. wikiData = data;
  869. document.getElementById('nSources').textContent = data.sources.length;
  870. document.getElementById('nEntities').textContent = data.entities.length;
  871. document.getElementById('nTopics').textContent = data.topics.length;
  872. renderList('sources', data.sources);
  873. renderList('entities', data.entities);
  874. renderList('topics', data.topics);
  875. renderList('analyses', data.analyses);
  876. const idxContent = data.index_content || 'Wiki is empty. Run wiki_compile.py to populate.';
  877. if (typeof marked !== 'undefined') {
  878. document.getElementById('pageContent').innerHTML = marked.parse(idxContent);
  879. } else {
  880. document.getElementById('pageContent').textContent = idxContent;
  881. }
  882. });
  883. }
  884. function renderList(cat, pages) {
  885. const el = document.getElementById('list-' + cat);
  886. el.innerHTML = pages.map(p =>
  887. '<div class="page-item" data-path="' + p.path + '" data-name="' + p.name + '" onclick="loadPage(\'' + p.path + '\',\'' + escAttr(p.name) + '\')">' +
  888. p.name.substring(0, 50) + '<span class="badge">' + (p.size/1024).toFixed(1) + 'K</span></div>'
  889. ).join('');
  890. }
  891. function loadPage(path, name) {
  892. document.querySelectorAll('.page-item').forEach(e => e.classList.remove('active'));
  893. document.querySelectorAll('[data-path="'+path+'"]').forEach(e => e.classList.add('active'));
  894. document.getElementById('pageTitle').textContent = name;
  895. document.getElementById('pageContent').innerHTML = '<div style="color:#aaa">Loading...</div>';
  896. fetch('/api/wiki/page/' + path).then(r=>r.json()).then(data => {
  897. if (typeof marked !== 'undefined') {
  898. document.getElementById('pageContent').innerHTML = marked.parse(data.content);
  899. } else {
  900. document.getElementById('pageContent').textContent = data.content;
  901. }
  902. });
  903. }
  904. function filterList(cat, q) {
  905. const items = document.querySelectorAll('#list-' + cat + ' .page-item');
  906. q = q.toLowerCase();
  907. items.forEach(el => { el.style.display = el.dataset.name.toLowerCase().includes(q) ? '' : 'none'; });
  908. }
  909. function escAttr(s) { return s.replace(/'/g, "\\'"); }
  910. loadWiki();
  911. </script>
  912. </body>
  913. </html>
  914. """
  915. RAG_HTML = r"""
  916. <!DOCTYPE html>
  917. <html lang="zh-CN">
  918. <head>
  919. <meta charset="UTF-8">
  920. <meta name="viewport" content="width=device-width, initial-scale=1.0">
  921. <title>RAG Chunks Browser</title>
  922. <style>
  923. * { margin:0; padding:0; box-sizing:border-box; }
  924. body { font-family: -apple-system,"PingFang SC","Microsoft YaHei",sans-serif; background:#f0f2f5; color:#333; }
  925. .header { background:linear-gradient(135deg,#1a3a5c 0%,#2d6aa0 100%); color:white; padding:16px 30px; }
  926. .header-inner { max-width:1400px; margin:0 auto; display:flex; justify-content:space-between; align-items:center; }
  927. .header h1 { font-size:22px; }
  928. .header a { color:white; text-decoration:none; opacity:0.8; }
  929. .header a:hover { opacity:1; }
  930. .container { max-width:1400px; margin:0 auto; padding:20px; }
  931. .stats-row { display:flex; gap:12px; margin-bottom:20px; }
  932. .stat-box { flex:1; background:white; border-radius:10px; padding:16px; text-align:center; box-shadow:0 1px 4px rgba(0,0,0,0.06); }
  933. .stat-num { font-size:32px; font-weight:700; color:#2d6aa0; }
  934. .stat-label { font-size:12px; color:#888; margin-top:4px; }
  935. .search-area { background:white; border-radius:12px; padding:20px; margin-bottom:20px; box-shadow:0 2px 8px rgba(0,0,0,0.08); }
  936. .search-row { display:flex; gap:12px; }
  937. .search-row input { flex:1; padding:12px 16px; border:2px solid #e0e0e0; border-radius:8px; font-size:16px; outline:none; }
  938. .search-row input:focus { border-color:#2d6aa0; }
  939. .btn { padding:12px 24px; background:#2d6aa0; color:white; border:none; border-radius:8px; font-size:15px; cursor:pointer; }
  940. .btn:hover { background:#1a5a8e; }
  941. .chunk-card { background:white; border-radius:10px; padding:16px; margin-bottom:12px; box-shadow:0 1px 4px rgba(0,0,0,0.06); border-left:4px solid #2d6aa0; }
  942. .chunk-header { display:flex; justify-content:space-between; margin-bottom:8px; font-size:13px; color:#888; }
  943. .chunk-source { font-weight:600; color:#2d6aa0; }
  944. .chunk-score { background:#e0eef8; padding:2px 8px; border-radius:10px; font-size:12px; }
  945. .chunk-text { font-size:14px; line-height:1.7; white-space:pre-wrap; word-wrap:break-word; max-height:200px; overflow-y:auto; background:#fafafa; padding:10px; border-radius:6px; }
  946. .sample-title { font-size:16px; font-weight:600; margin:20px 0 12px; color:#555; }
  947. </style>
  948. </head>
  949. <body>
  950. <div class="header">
  951. <div class="header-inner">
  952. <h1>RAG Chunks Browser</h1>
  953. <div><a href="/">QA Demo</a> | <a href="/wiki">Wiki</a></div>
  954. </div>
  955. </div>
  956. <div class="container">
  957. <div class="stats-row">
  958. <div class="stat-box"><div class="stat-num" id="nChunks">-</div><div class="stat-label">Total Chunks</div></div>
  959. <div class="stat-box"><div class="stat-num" id="nFiles">-</div><div class="stat-label">Source Files</div></div>
  960. <div class="stat-box"><div class="stat-num" id="nChunkSize">-</div><div class="stat-label">Chunk Size</div></div>
  961. <div class="stat-box"><div class="stat-num" id="nKeywords">-</div><div class="stat-label">Keywords</div></div>
  962. </div>
  963. <div class="search-area">
  964. <div class="search-row">
  965. <input type="text" id="searchInput" placeholder="Search RAG chunks (e.g. ship entry/exit reporting)...">
  966. <select id="topK" style="padding:10px;border-radius:8px;border:1px solid #ddd;font-size:14px;">
  967. <option value="5">Top 5</option>
  968. <option value="10" selected>Top 10</option>
  969. <option value="20">Top 20</option>
  970. <option value="50">Top 50</option>
  971. </select>
  972. <button class="btn" onclick="doSearch()">Search</button>
  973. </div>
  974. </div>
  975. <div id="results"></div>
  976. <div class="sample-title" id="sampleTitle">Sample Chunks (from index)</div>
  977. <div id="sampleChunks"></div>
  978. </div>
  979. <script>
  980. document.getElementById('searchInput').addEventListener('keydown', e => { if(e.key==='Enter') doSearch(); });
  981. function loadStats() {
  982. fetch('/api/rag/stats').then(r=>r.json()).then(data => {
  983. document.getElementById('nChunks').textContent = data.total_chunks.toLocaleString();
  984. document.getElementById('nFiles').textContent = data.total_files.toLocaleString();
  985. document.getElementById('nChunkSize').textContent = data.chunk_size;
  986. document.getElementById('nKeywords').textContent = data.keyword_index_size.toLocaleString();
  987. const el = document.getElementById('sampleChunks');
  988. el.innerHTML = data.sample_chunks.map(c => renderChunk(c, null)).join('');
  989. });
  990. }
  991. function doSearch() {
  992. const q = document.getElementById('searchInput').value.trim();
  993. if (!q) return;
  994. const k = document.getElementById('topK').value;
  995. document.getElementById('results').innerHTML = '<div style="text-align:center;padding:20px;color:#888;">Searching...</div>';
  996. document.getElementById('sampleTitle').style.display = 'none';
  997. document.getElementById('sampleChunks').style.display = 'none';
  998. fetch('/api/rag/search?q=' + encodeURIComponent(q) + '&k=' + k)
  999. .then(r=>r.json()).then(data => {
  1000. const el = document.getElementById('results');
  1001. if (!data.results.length) {
  1002. el.innerHTML = '<div style="text-align:center;padding:20px;color:#888;">No results found</div>';
  1003. return;
  1004. }
  1005. el.innerHTML = '<div style="margin-bottom:12px;font-size:14px;color:#666;">Found ' + data.results.length + ' chunks for "<b>' + escHtml(q) + '</b>"</div>' +
  1006. data.results.map(c => renderChunk(c, c.score)).join('');
  1007. });
  1008. }
  1009. function renderChunk(c, score) {
  1010. return '<div class="chunk-card">' +
  1011. '<div class="chunk-header"><span class="chunk-source">' + escHtml(c.source) +
  1012. ' #' + c.chunk_id + '</span>' +
  1013. (score !== null ? '<span class="chunk-score">Score: ' + score + '</span>' : '') +
  1014. '</div><div class="chunk-text">' + escHtml(c.text) + '</div></div>';
  1015. }
  1016. function escHtml(s) { const d=document.createElement('div'); d.textContent=s; return d.innerHTML; }
  1017. loadStats();
  1018. </script>
  1019. </body>
  1020. </html>
  1021. """
  1022. if __name__ == '__main__':
  1023. print("Loading search index...")
  1024. search("test", top_k=1) # warm up
  1025. print("Index loaded. Starting server...")
  1026. print("Access: http://" + DEMO_BIND + ":" + str(DEMO_PORT))
  1027. app.run(host=DEMO_BIND, port=DEMO_PORT, debug=False, threaded=True)