wiki_compile.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540
  1. #!/usr/bin/env python3
  2. """
  3. Wiki Full Compilation: Read domain docs → LLM understand → Write wiki pages
  4. Uses vLLM on NPU for fast inference
  5. For each document:
  6. 1. Read extracted text
  7. 2. LLM summarizes → write sources/<filename>.md
  8. 3. LLM extracts entities → create/update entities/<entity>.md
  9. 4. LLM identifies topics → create/update topics/<topic>.md
  10. 5. Update index.md and log.md
  11. Supports resume from interruption.
  12. """
  13. import json
  14. import os
  15. import re
  16. import sys
  17. import time
  18. import urllib.request
  19. from datetime import datetime
  20. from pathlib import Path
  21. from config import cfg
  22. PROCESSED_DIR = cfg.processed_dir
  23. WIKI_DIR = cfg.wiki_dir
  24. PROGRESS_FILE = WIKI_DIR / '.compile_progress.json'
  25. # vLLM API
  26. VLLM_URL = "http://127.0.0.1:8000/v1/chat/completions"
  27. # Fallback to Ollama if vLLM not available
  28. OLLAMA_URL = "http://127.0.0.1:8000/v1/chat/completions"
  29. API_URL = None # auto-detect
  30. def detect_api():
  31. global API_URL
  32. for url in [VLLM_URL, OLLAMA_URL]:
  33. try:
  34. test_body = json.dumps({
  35. "model": "qwen2.5-32b" if "8000" in url else "qwen2.5-32b",
  36. "messages": [{"role": "user", "content": "hi"}],
  37. "max_tokens": 5,
  38. }).encode('utf-8')
  39. req = urllib.request.Request(url, data=test_body,
  40. headers={"Content-Type": "application/json"}, method="POST")
  41. with urllib.request.urlopen(req, timeout=30) as resp:
  42. API_URL = url
  43. engine = "vLLM NPU" if "8000" in url else "Ollama CPU"
  44. print(f"Using: {engine} ({url})")
  45. return
  46. except:
  47. continue
  48. print("ERROR: No LLM API available!")
  49. sys.exit(1)
  50. def llm_chat(prompt, max_tokens=2048, temperature=0.2):
  51. model = "qwen2.5-32b" if "8000" in API_URL else "qwen2.5-32b"
  52. body = json.dumps({
  53. "model": model,
  54. "messages": [{"role": "user", "content": prompt}],
  55. "temperature": temperature,
  56. "max_tokens": max_tokens,
  57. }, ensure_ascii=False).encode('utf-8')
  58. req = urllib.request.Request(API_URL, data=body,
  59. headers={"Content-Type": "application/json"}, method="POST")
  60. with urllib.request.urlopen(req, timeout=300) as resp:
  61. data = json.loads(resp.read())
  62. return data["choices"][0]["message"]["content"].strip()
  63. def safe_filename(name, max_len=80):
  64. """Convert to safe filename"""
  65. name = re.sub(r'[\\/:*?"<>|]', '_', name)
  66. name = re.sub(r'\s+', '_', name)
  67. return name[:max_len]
  68. def load_progress():
  69. if PROGRESS_FILE.exists():
  70. with open(PROGRESS_FILE) as f:
  71. return json.load(f)
  72. return {'done': [], 'entities': {}, 'topics': {}, 'relations': []}
  73. def save_progress(progress):
  74. with open(PROGRESS_FILE, 'w', encoding='utf-8') as f:
  75. json.dump(progress, f, ensure_ascii=False, indent=2)
  76. def _extract_date_from_name(doc_name):
  77. """从文件名中提取日期信息"""
  78. # 匹配 "2017年8月31日起施行" 或 "2022年4月1日"
  79. m = re.search(r'(\d{4})年(\d{1,2})月(\d{1,2})日', doc_name)
  80. if m:
  81. return f"{m.group(1)}年{m.group(2)}月{m.group(3)}日"
  82. # 匹配 "2017年"
  83. m = re.search(r'(\d{4})年', doc_name)
  84. if m:
  85. return f"{m.group(1)}年"
  86. return "未知"
  87. def compile_source(doc_name, text, progress):
  88. """Step 1: Create source summary page"""
  89. # Truncate long documents
  90. text_sample = text[:6000] if len(text) > 6000 else text
  91. # Extract date from filename for temporal normalization
  92. doc_date = _extract_date_from_name(doc_name)
  93. prompt = f"""请阅读以下海事领域文档,输出结构化摘要。
  94. 文档名: {doc_name}
  95. 文档日期: {doc_date}
  96. 文档内容:
  97. {text_sample}
  98. 请按以下格式输出(不要输出其他内容):
  99. ## 摘要
  100. [2-4句话概括文档核心内容,必须包含具体的施行日期和文号]
  101. ## 关键信息
  102. - [要点1]
  103. - [要点2]
  104. - [要点3]
  105. (最多8个要点,每个要点尽量包含具体条款号、数字、日期)
  106. ## 时间线
  107. [列出文档中提到的所有时间节点,格式: 绝对日期|事件]
  108. {doc_date}|本文件施行
  109. (**重要**: 文档中的所有相对时间必须转换为绝对时间:
  110. - "自发布之日起施行" → "{doc_date}|施行"
  111. - "近年来" → 以{doc_date}为基准,推算为具体年份范围
  112. - "去年" → 以{doc_date}为基准,推算具体年份
  113. - "本条例修订前" → 写明具体的修订前版本年份
  114. - "三年内" → 从{doc_date}起算,写明截止的具体年份
  115. 禁止使用任何相对时间表述,全部转为绝对年月日)
  116. ## 实体列表
  117. [列出文档中涉及的重要实体,每行一个,格式: 类型|名称|说明]
  118. 法规|XXX法|(含施行日期和文号)
  119. 机构|XXX局|...
  120. 概念|XXX制度|...
  121. ## 实体关系
  122. [列出文档中实体之间的关系,每行一个,格式: 主体|关系|客体|说明]
  123. 交通运输部|制定|船员管理规定|交通运输部负责制定船员管理相关规定
  124. 海事局|隶属|交通运输部|海事局是交通运输部下属机构
  125. 船员适任证书|依据|STCW公约|船员适任证书制度依据STCW国际公约
  126. (列出文档中所有重要的实体间关系,包括:上下级/隶属、制定/发布、依据/引用、适用/管辖、修订/废止、时间先后等)
  127. ## 相关主题
  128. [列出文档涉及的主题领域,每行一个]
  129. 船员管理
  130. 船舶安全
  131. ...
  132. ## 元数据
  133. - 发布机构: [如有]
  134. - 施行日期: {doc_date}
  135. - 文号: [如有]
  136. - 法律层级: [全国人大/国务院/部委/海事局/其他]
  137. - 是否现行有效: [是/否/已被修订]
  138. - 修订历史: [如有,列出历次修订的绝对日期]"""
  139. result = llm_chat(prompt)
  140. # Write source page
  141. source_path = WIKI_DIR / 'sources' / f'{safe_filename(doc_name)}.md'
  142. source_path.parent.mkdir(parents=True, exist_ok=True)
  143. with open(source_path, 'w', encoding='utf-8') as f:
  144. f.write(f'# {doc_name}\n\n')
  145. f.write(f'[来源: {doc_name}]\n\n')
  146. f.write(result)
  147. f.write('\n')
  148. return result
  149. def extract_entities(llm_result, doc_name, progress):
  150. """Step 2: Extract and update entity pages"""
  151. entities_found = []
  152. lines = llm_result.split('\n')
  153. in_entity_section = False
  154. for line in lines:
  155. if '## 实体列表' in line:
  156. in_entity_section = True
  157. continue
  158. if line.startswith('## ') and in_entity_section:
  159. in_entity_section = False
  160. continue
  161. if in_entity_section and '|' in line:
  162. parts = [p.strip() for p in line.strip('- ').split('|')]
  163. if len(parts) >= 2:
  164. etype = parts[0]
  165. ename = parts[1]
  166. edesc = parts[2] if len(parts) > 2 else ''
  167. if ename and len(ename) > 1:
  168. entities_found.append((etype, ename, edesc))
  169. # Update entity pages
  170. for etype, ename, edesc in entities_found[:10]: # Max 10 per doc
  171. entity_file = safe_filename(ename)
  172. entity_path = WIKI_DIR / 'entities' / f'{entity_file}.md'
  173. entity_path.parent.mkdir(parents=True, exist_ok=True)
  174. if entity_path.exists():
  175. # Append reference
  176. existing = entity_path.read_text(encoding='utf-8')
  177. if doc_name not in existing:
  178. with open(entity_path, 'a', encoding='utf-8') as f:
  179. f.write(f'\n- {edesc} [来源: {doc_name}]\n')
  180. else:
  181. # Create new entity page
  182. with open(entity_path, 'w', encoding='utf-8') as f:
  183. f.write(f'# {ename}\n\n')
  184. f.write(f'**类型**: {etype}\n\n')
  185. f.write(f'## 描述\n\n{edesc}\n\n')
  186. f.write(f'## 相关文档\n\n')
  187. f.write(f'- {doc_name}\n')
  188. # Track in progress
  189. if ename not in progress['entities']:
  190. progress['entities'][ename] = {'type': etype, 'refs': []}
  191. if doc_name not in progress['entities'][ename]['refs']:
  192. progress['entities'][ename]['refs'].append(doc_name)
  193. return entities_found
  194. def extract_topics(llm_result, doc_name, progress):
  195. """Step 3: Extract and update topic pages"""
  196. topics_found = []
  197. lines = llm_result.split('\n')
  198. in_topic_section = False
  199. for line in lines:
  200. if '## 相关主题' in line:
  201. in_topic_section = True
  202. continue
  203. if line.startswith('## ') and in_topic_section:
  204. in_topic_section = False
  205. continue
  206. if in_topic_section:
  207. topic = line.strip('- ').strip()
  208. if topic and len(topic) > 1 and not topic.startswith('#'):
  209. topics_found.append(topic)
  210. # Update topic pages
  211. for topic in topics_found[:5]: # Max 5 per doc
  212. topic_file = safe_filename(topic)
  213. topic_path = WIKI_DIR / 'topics' / f'{topic_file}.md'
  214. topic_path.parent.mkdir(parents=True, exist_ok=True)
  215. if topic_path.exists():
  216. existing = topic_path.read_text(encoding='utf-8')
  217. if doc_name not in existing:
  218. with open(topic_path, 'a', encoding='utf-8') as f:
  219. f.write(f'- [[{safe_filename(doc_name)}]] - {doc_name}\n')
  220. else:
  221. with open(topic_path, 'w', encoding='utf-8') as f:
  222. f.write(f'# {topic}\n\n')
  223. f.write(f'## 相关文档\n\n')
  224. f.write(f'- [[{safe_filename(doc_name)}]] - {doc_name}\n')
  225. if topic not in progress['topics']:
  226. progress['topics'][topic] = []
  227. if doc_name not in progress['topics'][topic]:
  228. progress['topics'][topic].append(doc_name)
  229. return topics_found
  230. def extract_relations(llm_result, doc_name, progress):
  231. """Step 4: Extract entity relations and build knowledge graph"""
  232. relations_found = []
  233. lines = llm_result.split('\n')
  234. in_relation_section = False
  235. for line in lines:
  236. if '## 实体关系' in line:
  237. in_relation_section = True
  238. continue
  239. if line.startswith('## ') and in_relation_section:
  240. in_relation_section = False
  241. continue
  242. if in_relation_section and '|' in line:
  243. parts = [p.strip() for p in line.strip('- (').split('|')]
  244. if len(parts) >= 3:
  245. subject = parts[0]
  246. relation = parts[1]
  247. obj = parts[2]
  248. desc = parts[3] if len(parts) > 3 else ''
  249. if subject and relation and obj and len(subject) > 1:
  250. relations_found.append({
  251. 'subject': subject,
  252. 'relation': relation,
  253. 'object': obj,
  254. 'description': desc,
  255. 'source': doc_name,
  256. })
  257. # Store relations in progress
  258. if 'relations' not in progress:
  259. progress['relations'] = []
  260. for rel in relations_found[:15]: # Max 15 per doc
  261. # Deduplicate
  262. existing = any(
  263. r['subject'] == rel['subject'] and
  264. r['relation'] == rel['relation'] and
  265. r['object'] == rel['object']
  266. for r in progress['relations']
  267. )
  268. if not existing:
  269. progress['relations'].append(rel)
  270. # Update entity pages with relations
  271. for rel in relations_found[:15]:
  272. for ename in [rel['subject'], rel['object']]:
  273. entity_file = safe_filename(ename)
  274. entity_path = WIKI_DIR / 'entities' / f'{entity_file}.md'
  275. if entity_path.exists():
  276. existing = entity_path.read_text(encoding='utf-8')
  277. if '## 关系' not in existing:
  278. with open(entity_path, 'a', encoding='utf-8') as f:
  279. f.write(f'\n## 关系\n\n')
  280. rel_line = f'- {rel["subject"]} **{rel["relation"]}** {rel["object"]}'
  281. if rel['description']:
  282. rel_line += f' — {rel["description"]}'
  283. rel_line += f' [来源: {doc_name}]\n'
  284. existing = entity_path.read_text(encoding='utf-8')
  285. if rel_line.strip() not in existing:
  286. with open(entity_path, 'a', encoding='utf-8') as f:
  287. f.write(rel_line)
  288. # Write/update relations graph file
  289. relations_path = WIKI_DIR / 'relations.json'
  290. with open(relations_path, 'w', encoding='utf-8') as f:
  291. json.dump(progress['relations'], f, ensure_ascii=False, indent=2)
  292. return relations_found
  293. def update_index(progress):
  294. """Update wiki index.md"""
  295. sources = sorted(WIKI_DIR.glob('sources/*.md'))
  296. entities = sorted(WIKI_DIR.glob('entities/*.md'))
  297. topics = sorted(WIKI_DIR.glob('topics/*.md'))
  298. analyses = sorted(WIKI_DIR.glob('analyses/*.md'))
  299. index = f"""# Maritime Wiki Index
  300. > Auto-compiled from domain documents by wikiagent
  301. > Last updated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
  302. ## Sources ({len(sources)} files)
  303. """
  304. for s in sources[:50]: # Show first 50
  305. index += f'- [[{s.stem}]]\n'
  306. if len(sources) > 50:
  307. index += f'\n... and {len(sources) - 50} more\n'
  308. index += f'\n## Entities ({len(entities)})\n\n'
  309. # Group by type
  310. entity_types = {}
  311. for ename, einfo in progress.get('entities', {}).items():
  312. etype = einfo.get('type', 'other')
  313. if etype not in entity_types:
  314. entity_types[etype] = []
  315. entity_types[etype].append((ename, len(einfo.get('refs', []))))
  316. for etype, elist in sorted(entity_types.items()):
  317. index += f'### {etype} ({len(elist)})\n'
  318. for ename, ref_count in sorted(elist, key=lambda x: -x[1])[:20]:
  319. index += f'- [[{safe_filename(ename)}]] ({ref_count} refs)\n'
  320. if len(elist) > 20:
  321. index += f'- ... and {len(elist) - 20} more\n'
  322. index += '\n'
  323. index += f'## Topics ({len(topics)})\n\n'
  324. for tname, trefs in sorted(progress.get('topics', {}).items(), key=lambda x: -len(x[1]))[:30]:
  325. index += f'- [[{safe_filename(tname)}]] ({len(trefs)} docs)\n'
  326. if len(topics) > 30:
  327. index += f'\n... and {len(topics) - 30} more\n'
  328. index += f'\n## Analyses ({len(analyses)})\n\n'
  329. for a in analyses[:20]:
  330. index += f'- [[{a.stem}]]\n'
  331. # Relations summary
  332. relations = progress.get('relations', [])
  333. if relations:
  334. index += f'\n## Relations ({len(relations)})\n\n'
  335. # Group by relation type
  336. rel_types = {}
  337. for r in relations:
  338. rt = r['relation']
  339. if rt not in rel_types:
  340. rel_types[rt] = []
  341. rel_types[rt].append(r)
  342. for rt, rels in sorted(rel_types.items(), key=lambda x: -len(x[1]))[:15]:
  343. index += f'### {rt} ({len(rels)})\n'
  344. for r in rels[:5]:
  345. index += f'- {r["subject"]} → {r["object"]}\n'
  346. if len(rels) > 5:
  347. index += f'- ... and {len(rels) - 5} more\n'
  348. index += '\n'
  349. index += f'\n---\nStats: {len(sources)} sources, {len(entities)} entities, {len(topics)} topics, {len(relations)} relations, {len(analyses)} analyses\n'
  350. with open(WIKI_DIR / 'index.md', 'w', encoding='utf-8') as f:
  351. f.write(index)
  352. def update_log(doc_name, entities_count, topics_count, relations_count=0):
  353. """Append to log.md"""
  354. log_path = WIKI_DIR / 'log.md'
  355. timestamp = datetime.now().strftime('%Y-%m-%d %H:%M')
  356. entry = f'## [{timestamp}] Ingest | {doc_name}\n- Entities: {entities_count}, Topics: {topics_count}, Relations: {relations_count}\n\n'
  357. if log_path.exists():
  358. existing = log_path.read_text(encoding='utf-8')
  359. # Insert after header
  360. if '_no records_' in existing or '_no operations_' in existing:
  361. existing = '# Operation Log\n\n'
  362. with open(log_path, 'w', encoding='utf-8') as f:
  363. f.write(existing + entry)
  364. else:
  365. with open(log_path, 'w', encoding='utf-8') as f:
  366. f.write('# Operation Log\n\n' + entry)
  367. def main():
  368. detect_api()
  369. # Ensure wiki dirs exist
  370. for d in ['sources', 'entities', 'topics', 'analyses']:
  371. (WIKI_DIR / d).mkdir(parents=True, exist_ok=True)
  372. # Load progress
  373. progress = load_progress()
  374. done_set = set(progress['done'])
  375. # Get all text files
  376. all_docs = sorted(PROCESSED_DIR.glob('*.txt'))
  377. remaining = [d for d in all_docs if d.stem not in done_set]
  378. total = len(all_docs)
  379. print(f"Total documents: {total}")
  380. print(f"Already compiled: {len(done_set)}")
  381. print(f"Remaining: {len(remaining)}")
  382. print("=" * 60)
  383. start_time = time.time()
  384. errors = 0
  385. for idx, doc_path in enumerate(remaining):
  386. doc_name = doc_path.stem
  387. qnum = len(done_set) + idx + 1
  388. text = doc_path.read_text(encoding='utf-8')
  389. if len(text) < 50:
  390. print(f"[{qnum}/{total}] SKIP (too short): {doc_name[:60]}")
  391. progress['done'].append(doc_name)
  392. continue
  393. print(f"\n[{qnum}/{total}] {doc_name[:70]}...")
  394. try:
  395. # Step 1: Compile source summary
  396. t0 = time.time()
  397. llm_result = compile_source(doc_name, text, progress)
  398. t1 = time.time()
  399. print(f" Source page: {t1-t0:.1f}s")
  400. # Step 2: Extract entities
  401. entities = extract_entities(llm_result, doc_name, progress)
  402. print(f" Entities: {len(entities)} found")
  403. # Step 3: Extract topics
  404. topics = extract_topics(llm_result, doc_name, progress)
  405. print(f" Topics: {len(topics)} found")
  406. # Step 4: Extract relations
  407. relations = extract_relations(llm_result, doc_name, progress)
  408. print(f" Relations: {len(relations)} found")
  409. # Step 5: Update log
  410. update_log(doc_name, len(entities), len(topics), len(relations))
  411. progress['done'].append(doc_name)
  412. except Exception as e:
  413. errors += 1
  414. print(f" ERROR: {e}")
  415. # Still mark as done to avoid infinite retry
  416. progress['done'].append(doc_name)
  417. # Save progress and update index every 20 docs
  418. if (idx + 1) % 20 == 0 or idx == len(remaining) - 1:
  419. save_progress(progress)
  420. update_index(progress)
  421. elapsed = time.time() - start_time
  422. avg = elapsed / (idx + 1)
  423. eta = avg * (len(remaining) - idx - 1)
  424. print(f"\n [Progress] {qnum}/{total} done | "
  425. f"Elapsed: {elapsed/60:.1f}min | ETA: {eta/60:.1f}min | "
  426. f"Errors: {errors} | "
  427. f"Entities: {len(progress['entities'])} | "
  428. f"Topics: {len(progress['topics'])} | "
  429. f"Relations: {len(progress.get('relations', []))}")
  430. # Final update
  431. save_progress(progress)
  432. update_index(progress)
  433. total_time = time.time() - start_time
  434. sources = len(list(WIKI_DIR.glob('sources/*.md')))
  435. entities = len(list(WIKI_DIR.glob('entities/*.md')))
  436. topics = len(list(WIKI_DIR.glob('topics/*.md')))
  437. print(f"\n{'=' * 60}")
  438. print(f"Wiki Compilation Complete!")
  439. print(f" Time: {total_time/60:.1f} minutes")
  440. print(f" Sources: {sources}")
  441. print(f" Entities: {entities}")
  442. print(f" Topics: {topics}")
  443. print(f" Errors: {errors}")
  444. print(f" Wiki dir: {WIKI_DIR}")
  445. if __name__ == '__main__':
  446. main()