knowledge.py 72 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931
  1. """
  2. api.v1.knowledge — Knowledge Base management endpoints.
  3. GET /api/v1/knowledge List all KBs
  4. POST /api/v1/knowledge Create KB (name, root_dir)
  5. DELETE /api/v1/knowledge/{kb_id} Delete KB
  6. GET /api/v1/knowledge/{kb_id}/files List root_dir file tree
  7. POST /api/v1/knowledge/{kb_id}/files Add files to KB
  8. DELETE /api/v1/knowledge/{kb_id}/files/{file_id} Remove file from KB
  9. POST /api/v1/knowledge/{kb_id}/index Trigger index build job
  10. POST /api/v1/knowledge/{kb_id}/wiki Trigger wiki compile job
  11. PUT /api/v1/knowledge/{kb_id}/wiki/pause Pause/resume wiki compile
  12. GET /api/v1/knowledge/{kb_id}/wiki/tree Get wiki file tree
  13. GET /api/v1/knowledge/{kb_id}/wiki/{page_path} Get wiki page content
  14. POST /api/v1/knowledge/{kb_id}/search Search KB
  15. GET /api/v1/knowledge/{kb_id}/search/stream SSE streaming QA
  16. GET /api/v1/knowledge/{kb_id}/jobs List jobs for KB
  17. GET /api/v1/knowledge/{kb_id}/jobs/{job_id} Get job status + log
  18. """
  19. from __future__ import annotations
  20. import asyncio
  21. import json
  22. import os
  23. import subprocess
  24. import sys
  25. import tempfile
  26. import time
  27. import threading
  28. from pathlib import Path
  29. from typing import Any, Dict, List, Optional, Set
  30. from fastapi import APIRouter, Depends, HTTPException, Query
  31. from fastapi.responses import StreamingResponse
  32. from pydantic import BaseModel, Field
  33. from agentpaas.api.deps import get_tenant, get_database
  34. from agentpaas.api.errors import api_error
  35. from agentpaas.api.middleware.auth import TenantContext
  36. from agentpaas.db.models import Database, gen_id, now_utc
  37. router = APIRouter(prefix="/knowledge", tags=["knowledge"])
  38. # ─── Supported file extensions ───
  39. SUPPORTED_EXTS = {".pdf", ".txt", ".docx", ".md", ".html", ".htm", ".csv"}
  40. # ─── Pydantic models ───
  41. class CreateKBRequest(BaseModel):
  42. name: str = Field(..., max_length=256)
  43. root_dir: str = Field(..., description="Absolute path to knowledge base directory")
  44. description: str = Field(default="", max_length=2048)
  45. class AddFilesRequest(BaseModel):
  46. file_paths: List[str] = Field(..., description="Absolute paths of files to add")
  47. class IndexBuildRequest(BaseModel):
  48. index_type: str = Field(default="bm25", description="bm25 | vector | graph | all")
  49. mode: str = Field(default="full", description="full | incremental")
  50. chunk_strategy: str = Field(default="token", description="token | contextual | page")
  51. class WikiBuildRequest(BaseModel):
  52. mode: str = Field(default="incremental", description="incremental | rebuild")
  53. concurrency: int = Field(default=2, ge=1, le=8)
  54. class SearchRequest(BaseModel):
  55. query: str = Field(..., max_length=4096)
  56. mode: str = Field(default="lambda", description="lambda | bm25 | vector | wiki | fusion | pageindex")
  57. top_k: int = Field(default=5, ge=1, le=20)
  58. weights: Optional[Dict[str, float]] = None # bm25/vector/graph/wiki overrides
  59. class UpdateIgnoreDirsRequest(BaseModel):
  60. ignored_dirs: List[str] = Field(default_factory=list, description="List of relative dir paths to ignore")
  61. class WikiScheduleRequest(BaseModel):
  62. enabled: bool = True
  63. interval: str = Field(default="weekly", description="daily | weekly")
  64. # ─── Helper: walk root_dir ───
  65. def _walk_dir(
  66. root_dir: str,
  67. added_paths: set,
  68. ignored_dirs: Optional[Set[str]] = None,
  69. subpath: str = "",
  70. name_filter: str = "",
  71. limit: int = 500,
  72. ) -> List[Dict]:
  73. """
  74. Recursively list files under root_dir (or subpath within root_dir).
  75. - ignored_dirs: set of relative paths to skip
  76. - subpath: only scan this subdirectory (relative to root_dir)
  77. - name_filter: case-insensitive filename substring filter
  78. - limit: max results (default 500, was 2000)
  79. """
  80. ignored = ignored_dirs or set()
  81. items: List[Dict] = []
  82. root = Path(root_dir)
  83. scan_root = (root / subpath) if subpath else root
  84. if not scan_root.is_dir():
  85. return items
  86. name_lower = name_filter.lower() if name_filter else ""
  87. for dirpath, dirnames, filenames in os.walk(str(scan_root), topdown=True):
  88. rel_dir = Path(dirpath).relative_to(root)
  89. parts = rel_dir.parts # () for root itself
  90. # If any ancestor or the current dir matches the ignore list, skip entire subtree
  91. if parts:
  92. ignored_hit = any(
  93. parts[i] in ignored or "/".join(parts[:i + 1]) in ignored
  94. for i in range(len(parts))
  95. )
  96. if ignored_hit:
  97. dirnames.clear()
  98. continue
  99. # Prune child dirs that match ignore list
  100. dirnames[:] = sorted(
  101. d for d in dirnames
  102. if d not in ignored and (
  103. "/".join((*parts, d)) if parts else d
  104. ) not in ignored
  105. )
  106. for fname in sorted(filenames):
  107. if name_lower and name_lower not in fname.lower():
  108. continue
  109. p = Path(dirpath) / fname
  110. if p.suffix.lower() not in SUPPORTED_EXTS:
  111. continue
  112. rel_file = str(p.relative_to(root))
  113. try:
  114. stat = p.stat()
  115. except OSError:
  116. continue
  117. items.append({
  118. "path": str(p),
  119. "relative_path": rel_file,
  120. "name": fname,
  121. "type": p.suffix.lower().lstrip("."),
  122. "size": stat.st_size,
  123. "modified": stat.st_mtime,
  124. "added": str(p) in added_paths,
  125. })
  126. if len(items) >= limit:
  127. return items
  128. return items
  129. def _list_dir_shallow(root_dir: str, parent_rel: str, ignored: Set[str]) -> List[Dict]:
  130. """
  131. List ONE level of subdirectories under parent_rel (relative to root_dir).
  132. Returns nodes with empty children=[]; has_children indicates whether the
  133. node has any subdirectory children so the frontend can show an expand arrow.
  134. This replaces the full-depth _list_dir_tree to support lazy loading.
  135. """
  136. root = Path(root_dir)
  137. scan_path = (root / parent_rel) if parent_rel else root
  138. result = []
  139. try:
  140. for child in sorted(scan_path.iterdir()):
  141. if not child.is_dir():
  142. continue
  143. child_rel = f"{parent_rel}/{child.name}" if parent_rel else child.name
  144. self_ignored = child.name in ignored or child_rel in ignored
  145. # Peek one level to know if there are subdirs (cheap: stops at first hit)
  146. has_children = False
  147. try:
  148. for gc in child.iterdir():
  149. if gc.is_dir():
  150. has_children = True
  151. break
  152. except OSError:
  153. pass
  154. result.append({
  155. "name": child.name,
  156. "relative_path": child_rel,
  157. "ignored": self_ignored,
  158. "has_children": has_children,
  159. "children": [], # populated lazily by the frontend
  160. })
  161. except OSError:
  162. pass
  163. return result
  164. # ─── Helper: run index build in background ───
  165. def _write_instance_yml(kb_root: str) -> str:
  166. """Write a temporary instance.yml that pins all config paths to kb_root.
  167. config.py loads agent-config.yml first (which may contain hardcoded remote
  168. paths like /home/67/knowledge/...) and then overlays instance.yml via deep
  169. merge. By setting INSTANCE_CONFIG to this temp file we override those paths
  170. with the actual local kb_root before any script reads cfg.base_dir.
  171. Returns the temp file path; caller is responsible for os.unlink() cleanup.
  172. """
  173. kb = Path(kb_root)
  174. # Single-quote YAML scalars — safe for paths without single quotes
  175. lines = [
  176. "knowledge:",
  177. f" baseDir: '{kb_root}'",
  178. f" rawDir: '{kb_root}'",
  179. f" processedDir: '{kb / 'processed'}'",
  180. f" indexFile: '{kb / 'rag_index.json'}'",
  181. f" keywordFile: '{kb / 'rag_index.keywords.json'}'",
  182. f" vectorIndex: '{kb / 'rag_vectors_v2.npy'}'",
  183. f" vectorMeta: '{kb / 'rag_vectors_meta_v2.pkl'}'",
  184. "wiki:",
  185. f" dir: '{kb / 'wiki'}'",
  186. ]
  187. with tempfile.NamedTemporaryFile(mode='w', suffix='.yml', delete=False) as f:
  188. f.write("\n".join(lines) + "\n")
  189. return f.name
  190. def _make_script_env(kb_root: str, scripts_dir: Path, extra: Optional[Dict[str, str]] = None) -> Dict[str, str]:
  191. """Build subprocess env: inherit os.environ + KNOWLEDGE_DIR + PYTHONPATH.
  192. Does NOT write the instance.yml here (caller manages temp file lifecycle).
  193. Pass INSTANCE_CONFIG via `extra` if needed.
  194. """
  195. env = dict(os.environ)
  196. env["KNOWLEDGE_DIR"] = kb_root
  197. # Ensure scripts_dir is in PYTHONPATH so `from config import cfg` works
  198. existing_py = env.get("PYTHONPATH", "")
  199. env["PYTHONPATH"] = str(scripts_dir) + (os.pathsep + existing_py if existing_py else "")
  200. if extra:
  201. env.update(extra)
  202. return env
  203. def _popen_stream(
  204. script: Path,
  205. env: Dict[str, str],
  206. cwd: str,
  207. log_buf: List[str],
  208. db: Database,
  209. job_id: str,
  210. flush_every: int = 8,
  211. timeout: int = 3600,
  212. ) -> int:
  213. """Run script with Popen, stream stdout to log_buf, flush to DB every N lines.
  214. Returns process returncode. stderr is appended to log_buf on exit.
  215. """
  216. proc = subprocess.Popen(
  217. [sys.executable, str(script)],
  218. cwd=cwd, env=env,
  219. stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  220. text=True, bufsize=1,
  221. )
  222. flush_count = 0
  223. try:
  224. for line in proc.stdout: # type: ignore[union-attr]
  225. log_buf.append(line.rstrip("\n"))
  226. flush_count += 1
  227. if flush_count % flush_every == 0:
  228. try:
  229. db.execute(
  230. "UPDATE kb_jobs SET log=? WHERE id=?",
  231. ("\n".join(log_buf)[-8000:], job_id),
  232. )
  233. db.commit()
  234. except Exception:
  235. pass
  236. proc.wait(timeout=timeout)
  237. except subprocess.TimeoutExpired:
  238. proc.kill()
  239. log_buf.append("[超时] 进程已终止")
  240. stderr_out = (proc.stderr.read() or "").strip() # type: ignore[union-attr]
  241. if stderr_out:
  242. log_buf.append("--- stderr ---")
  243. log_buf.extend(stderr_out[-2000:].splitlines())
  244. return proc.returncode
  245. def _run_index_job(db: Database, kb_id: str, job_id: str, kb_root: str, index_type: str, mode: str, chunk_strategy: str = "token"):
  246. """Background thread — extract text then build index, streaming log to DB."""
  247. def _flush(status: str, log_buf: List[str]):
  248. try:
  249. db.execute(
  250. "UPDATE kb_jobs SET status=?, log=? WHERE id=?",
  251. (status, "\n".join(log_buf)[-8000:], job_id),
  252. )
  253. db.commit()
  254. except Exception:
  255. pass
  256. kb_path = Path(kb_root)
  257. scripts_dir = _find_scripts_dir(kb_path)
  258. instance_yml = _write_instance_yml(kb_root) if scripts_dir else None
  259. log_buf: List[str] = []
  260. errors: List[str] = []
  261. _flush("running", log_buf)
  262. try:
  263. if index_type in ("bm25", "all"):
  264. if not scripts_dir:
  265. log_buf.append("[BM25] 脚本目录未找到 — 已跳过")
  266. else:
  267. env = _make_script_env(kb_root, scripts_dir, {"INSTANCE_CONFIG": instance_yml})
  268. processed_dir = kb_path / "processed"
  269. # ── Step 1: 文本提取 ──
  270. extract_script = scripts_dir / "extract_docs.py"
  271. if extract_script.exists():
  272. existing_txts = list(processed_dir.glob("*.txt")) if processed_dir.is_dir() else []
  273. if not existing_txts or mode == "full":
  274. log_buf.append("[提取] 正在提取原始文档为文本...")
  275. _flush("running", log_buf)
  276. rc = _popen_stream(extract_script, env, kb_root, log_buf, db, job_id, flush_every=10)
  277. if rc != 0:
  278. errors.append("文本提取失败(returncode={rc})")
  279. else:
  280. log_buf.append(f"[提取] 已有 {len(existing_txts)} 个文本文件,跳过(全量模式可强制重新提取)")
  281. else:
  282. log_buf.append("[提取] extract_docs.py 未找到,跳过(需手动准备 processed/ 目录)")
  283. # ── Step 2: 构建 BM25 索引 ──
  284. bm25_script = next(
  285. (scripts_dir / n for n in ("rebuild_index.py", "build_index.py") if (scripts_dir / n).exists()),
  286. None,
  287. )
  288. if bm25_script:
  289. log_buf.append(f"[BM25] 正在构建索引({bm25_script.name})...")
  290. _flush("running", log_buf)
  291. rc = _popen_stream(bm25_script, env, kb_root, log_buf, db, job_id, flush_every=5)
  292. if rc != 0:
  293. errors.append(f"BM25 构建失败(returncode={rc})")
  294. elif chunk_strategy == "contextual":
  295. # ── Step 2b: Contextual chunking enrichment ──
  296. log_buf.append("[Contextual] 启动上下文增强分块...")
  297. _flush("running", log_buf)
  298. _contextual_enrich_index(kb_path, scripts_dir, log_buf, db, job_id, _flush)
  299. elif chunk_strategy == "page":
  300. # ── Step 2c: PageIndex build (page-level, no vector) ──
  301. log_buf.append("[PageIndex] 构建页面级索引...")
  302. _flush("running", log_buf)
  303. _build_page_index(kb_path, log_buf, db, job_id, _flush)
  304. else:
  305. log_buf.append("[BM25] rebuild_index.py / build_index.py 均未找到 — 已跳过")
  306. if index_type in ("vector", "all"):
  307. if not scripts_dir:
  308. log_buf.append("[Vector] 脚本目录未找到 — 已跳过")
  309. else:
  310. env = _make_script_env(kb_root, scripts_dir, {"INSTANCE_CONFIG": instance_yml})
  311. vec_script = scripts_dir / "build_vector_index.py"
  312. if vec_script.exists():
  313. log_buf.append("[Vector] 正在构建向量索引...")
  314. _flush("running", log_buf)
  315. rc = _popen_stream(vec_script, env, kb_root, log_buf, db, job_id, flush_every=5, timeout=3600)
  316. if rc != 0:
  317. errors.append(f"向量索引失败(returncode={rc})")
  318. else:
  319. log_buf.append("[Vector] build_vector_index.py 未找到 — 已跳过")
  320. if index_type in ("graph", "all"):
  321. log_buf.append("[Graph] 图谱由 wiki_compile.py 生成 wiki/relations.json — 请先运行 Wiki 编译")
  322. if errors:
  323. log_buf.append("\n❌ ERRORS:\n" + "\n".join(errors))
  324. _flush("failed" if errors else "completed", log_buf)
  325. db.execute("UPDATE kb_jobs SET completed_at=? WHERE id=?", (now_utc(), job_id))
  326. db.commit()
  327. except Exception as exc:
  328. log_buf.append(f"\n❌ 异常: {exc}")
  329. _flush("failed", log_buf)
  330. db.execute(
  331. "UPDATE kb_jobs SET error=?, completed_at=? WHERE id=?",
  332. (str(exc)[:500], now_utc(), job_id),
  333. )
  334. db.commit()
  335. finally:
  336. if instance_yml:
  337. try:
  338. os.unlink(instance_yml)
  339. except Exception:
  340. pass
  341. def _detect_local_llm_url() -> str:
  342. """Probe common local LLM endpoints and return the first that responds.
  343. Priority: VLLM_URL env var → Ollama :11434 → Ollama :11435 → vLLM :8001
  344. Falls back to the env default so we never block on a missing service here.
  345. """
  346. import urllib.request as _ur
  347. # If explicitly set by operator/user, trust it
  348. if os.environ.get("VLLM_URL"):
  349. return os.environ["VLLM_URL"]
  350. candidates = [
  351. "http://127.0.0.1:11434/v1/chat/completions", # Ollama default
  352. "http://127.0.0.1:11435/v1/chat/completions", # Ollama alt port
  353. "http://127.0.0.1:8001/v1/chat/completions", # vLLM on non-conflicting port
  354. ]
  355. for url in candidates:
  356. try:
  357. req = _ur.Request(url, data=b'{}', headers={"Content-Type": "application/json"}, method="POST")
  358. _ur.urlopen(req, timeout=2)
  359. except Exception as exc:
  360. # Connection refused / timeout → not available; 4xx means server IS there
  361. msg = str(exc)
  362. if "400" in msg or "422" in msg or "405" in msg or "404" in msg:
  363. return url
  364. else:
  365. return url # 2xx on empty payload — unlikely but accept
  366. # Nothing found; return Ollama default and let wiki_compile.py fail with a clear message
  367. return "http://127.0.0.1:11434/v1/chat/completions"
  368. def _run_wiki_job(db: Database, kb_id: str, job_id: str, kb_root: str, mode: str, concurrency: int):
  369. """Background thread: run wiki_compile.py with live log streaming to DB."""
  370. def _flush(status: str, log_buf: List[str]):
  371. try:
  372. db.execute(
  373. "UPDATE kb_jobs SET status=?, log=? WHERE id=?",
  374. (status, "\n".join(log_buf)[-8000:], job_id),
  375. )
  376. db.commit()
  377. except Exception:
  378. pass
  379. kb_path = Path(kb_root)
  380. scripts_dir = _find_scripts_dir(kb_path)
  381. script = scripts_dir / "wiki_compile.py" if scripts_dir else None
  382. instance_yml = _write_instance_yml(kb_root) if scripts_dir else None
  383. log_buf: List[str] = []
  384. _flush("running", log_buf)
  385. try:
  386. if not script or not script.exists():
  387. db.execute(
  388. "UPDATE kb_jobs SET status='failed', error=?, completed_at=? WHERE id=?",
  389. ("wiki_compile.py 未在脚本目录中找到", now_utc(), job_id),
  390. )
  391. db.commit()
  392. return
  393. llm_url = _detect_local_llm_url()
  394. log_buf.append(f"[Wiki] LLM 端点: {llm_url}")
  395. _flush("running", log_buf)
  396. env = _make_script_env(kb_root, scripts_dir, {
  397. "INSTANCE_CONFIG": instance_yml,
  398. "WIKI_DIR": str(kb_path / "wiki"),
  399. "VLLM_URL": llm_url,
  400. })
  401. if mode == "rebuild":
  402. env["WIKI_REBUILD"] = "1"
  403. env["WIKI_CONCURRENCY"] = str(concurrency)
  404. rc = _popen_stream(script, env, kb_root, log_buf, db, job_id, flush_every=3, timeout=7200)
  405. _flush("failed" if rc != 0 else "completed", log_buf)
  406. db.execute("UPDATE kb_jobs SET completed_at=? WHERE id=?", (now_utc(), job_id))
  407. db.commit()
  408. except Exception as exc:
  409. log_buf.append(f"\n❌ 异常: {exc}")
  410. _flush("failed", log_buf)
  411. db.execute(
  412. "UPDATE kb_jobs SET error=?, completed_at=? WHERE id=?",
  413. (str(exc)[:500], now_utc(), job_id),
  414. )
  415. db.commit()
  416. finally:
  417. if instance_yml:
  418. try:
  419. os.unlink(instance_yml)
  420. except Exception:
  421. pass
  422. def _find_scripts_dir(kb_path: Path) -> Optional[Path]:
  423. """Locate the scripts dir containing rebuild_index.py / wiki_compile.py etc.
  424. Checks kb_path itself (if scripts live alongside data) and the canonical
  425. project path agentexample/qaagent67lambda/scripts/.
  426. Uses rebuild_index.py or wiki_compile.py as the presence marker
  427. (build_index.py is the old hardcoded variant and no longer required).
  428. """
  429. _markers = ("rebuild_index.py", "wiki_compile.py", "build_index.py")
  430. # Check if scripts live inside kb_path or its parent
  431. for cand in [kb_path, kb_path.parent]:
  432. if any((cand / m).exists() for m in _markers):
  433. return cand
  434. # Walk up from this source file looking for the canonical scripts dir
  435. here = Path(__file__).resolve()
  436. for _ in range(8):
  437. here = here.parent
  438. cand = here / "agentexample" / "qaagent67lambda" / "scripts"
  439. if cand.is_dir() and any((cand / m).exists() for m in _markers):
  440. return cand
  441. return None
  442. def _get_index_status(kb_root: str) -> Dict[str, Any]:
  443. """Read index file presence/stats from kb_root."""
  444. root = Path(kb_root)
  445. bm25_file = root / "rag_index.json"
  446. vector_file = root / "rag_vectors_v2.npy"
  447. wiki_dir = root / "wiki"
  448. def _fstat(p: Path) -> Optional[Dict]:
  449. if p.exists():
  450. s = p.stat()
  451. return {"exists": True, "size": s.st_size, "mtime": s.st_mtime}
  452. return {"exists": False}
  453. bm25 = _fstat(bm25_file)
  454. if bm25["exists"]:
  455. try:
  456. import json as _json
  457. with open(bm25_file) as f:
  458. idx = _json.load(f)
  459. bm25["total_chunks"] = idx.get("total_chunks", 0)
  460. bm25["total_files"] = idx.get("total_files", 0)
  461. except Exception:
  462. pass
  463. vector = _fstat(vector_file)
  464. wiki = {"exists": wiki_dir.is_dir()}
  465. if wiki["exists"]:
  466. subdirs = {}
  467. for sub in ["sources", "entities", "topics", "analyses"]:
  468. d = wiki_dir / sub
  469. subdirs[sub] = len(list(d.glob("*.md"))) if d.is_dir() else 0
  470. wiki["subdirs"] = subdirs
  471. wiki["total_pages"] = sum(subdirs.values())
  472. return {"bm25": bm25, "vector": vector, "wiki": wiki}
  473. # ─── Chunk index cache ───
  474. # Avoid re-reading a 17 MB JSON on every request; invalidate when file mtime changes.
  475. _chunk_cache: Dict[str, Any] = {} # kb_id → {"mtime": float, "chunks": list, "total": int}
  476. def _load_chunks(kb_root: str, kb_id: str) -> Optional[List[Dict]]:
  477. """Load chunks from rag_index.json with mtime-based in-process cache."""
  478. index_file = Path(kb_root) / "rag_index.json"
  479. if not index_file.exists():
  480. return None
  481. mtime = index_file.stat().st_mtime
  482. cached = _chunk_cache.get(kb_id)
  483. if cached and cached["mtime"] == mtime:
  484. return cached["chunks"]
  485. try:
  486. with open(index_file, encoding="utf-8") as f:
  487. data = json.load(f)
  488. chunks = data.get("chunks", [])
  489. _chunk_cache[kb_id] = {"mtime": mtime, "chunks": chunks, "total": len(chunks)}
  490. return chunks
  491. except Exception:
  492. return None
  493. # ─── CRUD Endpoints ───
  494. @router.get("")
  495. async def list_kbs(
  496. tenant: TenantContext = Depends(get_tenant),
  497. db: Database = Depends(get_database),
  498. ):
  499. rows = db.fetchall(
  500. "SELECT * FROM knowledge_bases WHERE tenant_id=? ORDER BY created_at DESC",
  501. (tenant.tenant_id,)
  502. )
  503. result = []
  504. for r in rows:
  505. status = _get_index_status(r["root_dir"])
  506. result.append({**r, "index_status": status})
  507. return {"items": result}
  508. @router.post("", status_code=201)
  509. async def create_kb(
  510. req: CreateKBRequest,
  511. tenant: TenantContext = Depends(get_tenant),
  512. db: Database = Depends(get_database),
  513. ):
  514. root = Path(req.root_dir)
  515. if not root.is_dir():
  516. raise HTTPException(status_code=400, detail=f"Directory not found: {req.root_dir}")
  517. kb_id = gen_id("kb_")
  518. db.execute(
  519. "INSERT INTO knowledge_bases (id, tenant_id, name, root_dir, description, created_at, updated_at) "
  520. "VALUES (?, ?, ?, ?, ?, ?, ?)",
  521. (kb_id, tenant.tenant_id, req.name, str(root.resolve()), req.description, now_utc(), now_utc())
  522. )
  523. db.commit()
  524. return db.fetchone("SELECT * FROM knowledge_bases WHERE id=?", (kb_id,))
  525. @router.get("/{kb_id}")
  526. async def get_kb(
  527. kb_id: str,
  528. tenant: TenantContext = Depends(get_tenant),
  529. db: Database = Depends(get_database),
  530. ):
  531. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  532. if not kb:
  533. raise HTTPException(status_code=404, detail="Knowledge base not found")
  534. status = _get_index_status(kb["root_dir"])
  535. return {**kb, "index_status": status}
  536. @router.delete("/{kb_id}", status_code=204)
  537. async def delete_kb(
  538. kb_id: str,
  539. tenant: TenantContext = Depends(get_tenant),
  540. db: Database = Depends(get_database),
  541. ):
  542. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  543. if not kb:
  544. raise HTTPException(status_code=404, detail="Knowledge base not found")
  545. db.execute("DELETE FROM kb_files WHERE kb_id=?", (kb_id,))
  546. db.execute("DELETE FROM kb_jobs WHERE kb_id=?", (kb_id,))
  547. db.execute("DELETE FROM knowledge_bases WHERE id=?", (kb_id,))
  548. db.commit()
  549. return None
  550. # ─── Chunk Browser ───
  551. @router.get("/{kb_id}/chunks")
  552. async def list_chunks(
  553. kb_id: str,
  554. q: str = Query(default="", description="Keyword filter (case-insensitive substring)"),
  555. source: str = Query(default="", description="Filter by source file name"),
  556. page: int = Query(default=1, ge=1),
  557. limit: int = Query(default=20, ge=1, le=100),
  558. tenant: TenantContext = Depends(get_tenant),
  559. db: Database = Depends(get_database),
  560. ):
  561. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  562. if not kb:
  563. raise HTTPException(status_code=404, detail="Knowledge base not found")
  564. chunks = _load_chunks(kb["root_dir"], kb_id)
  565. if chunks is None:
  566. raise HTTPException(status_code=404, detail="BM25 index not built yet (rag_index.json not found)")
  567. # Filter
  568. q_lower = q.strip().lower()
  569. src_lower = source.strip().lower()
  570. filtered = chunks
  571. if q_lower:
  572. filtered = [c for c in filtered if q_lower in c.get("text", "").lower()]
  573. if src_lower:
  574. filtered = [c for c in filtered if src_lower in c.get("source", "").lower()]
  575. total = len(filtered)
  576. offset = (page - 1) * limit
  577. page_chunks = filtered[offset: offset + limit]
  578. # Return trimmed chunks (avoid huge payloads)
  579. return {
  580. "total": total,
  581. "total_all": len(chunks),
  582. "page": page,
  583. "limit": limit,
  584. "pages": max(1, (total + limit - 1) // limit),
  585. "chunks": [
  586. {
  587. "id": c.get("id", ""),
  588. "source": c.get("source", ""),
  589. "chunk_id": c.get("chunk_id", 0),
  590. "text": c.get("text", ""),
  591. "length": c.get("length", 0),
  592. }
  593. for c in page_chunks
  594. ],
  595. }
  596. # ─── File Management ───
  597. @router.get("/{kb_id}/files")
  598. async def list_files(
  599. kb_id: str,
  600. path: str = Query(default="", description="Subdirectory relative path to list files in"),
  601. q: str = Query(default="", description="Filename filter (case-insensitive substring)"),
  602. tenant: TenantContext = Depends(get_tenant),
  603. db: Database = Depends(get_database),
  604. ):
  605. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  606. if not kb:
  607. raise HTTPException(status_code=404, detail="Knowledge base not found")
  608. # Load saved ignore list
  609. raw_ignored = kb.get("ignore_dirs") or "[]"
  610. try:
  611. ignored: Set[str] = set(json.loads(raw_ignored))
  612. except Exception:
  613. ignored = set()
  614. added = db.fetchall("SELECT file_path FROM kb_files WHERE kb_id=?", (kb_id,))
  615. added_paths = {r["file_path"] for r in added}
  616. root_dir = kb["root_dir"]
  617. # Run blocking I/O off the event loop
  618. files, dirs = await asyncio.get_event_loop().run_in_executor(
  619. None,
  620. lambda: (
  621. _walk_dir(root_dir, added_paths, ignored, subpath=path, name_filter=q),
  622. _list_dir_shallow(root_dir, path, ignored),
  623. )
  624. )
  625. return {
  626. "kb_id": kb_id,
  627. "root_dir": root_dir,
  628. "files": files,
  629. "dirs": dirs,
  630. "ignored_dirs": sorted(ignored),
  631. "current_path": path,
  632. }
  633. @router.get("/{kb_id}/dirs")
  634. async def list_dirs(
  635. kb_id: str,
  636. path: str = Query(default="", description="Parent directory relative path"),
  637. tenant: TenantContext = Depends(get_tenant),
  638. db: Database = Depends(get_database),
  639. ):
  640. """Return immediate subdirectories of `path` (lazy tree expansion)."""
  641. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  642. if not kb:
  643. raise HTTPException(status_code=404, detail="Knowledge base not found")
  644. raw_ignored = kb.get("ignore_dirs") or "[]"
  645. try:
  646. ignored: Set[str] = set(json.loads(raw_ignored))
  647. except Exception:
  648. ignored = set()
  649. dirs = await asyncio.get_event_loop().run_in_executor(
  650. None,
  651. lambda: _list_dir_shallow(kb["root_dir"], path, ignored),
  652. )
  653. return {"dirs": dirs, "path": path}
  654. @router.get("/{kb_id}/ignore-dirs")
  655. async def get_ignore_dirs(
  656. kb_id: str,
  657. tenant: TenantContext = Depends(get_tenant),
  658. db: Database = Depends(get_database),
  659. ):
  660. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  661. if not kb:
  662. raise HTTPException(status_code=404, detail="Knowledge base not found")
  663. raw = kb.get("ignore_dirs") or "[]"
  664. try:
  665. ignored = sorted(set(json.loads(raw)))
  666. except Exception:
  667. ignored = []
  668. return {"ignored_dirs": ignored}
  669. @router.put("/{kb_id}/ignore-dirs")
  670. async def update_ignore_dirs(
  671. kb_id: str,
  672. req: UpdateIgnoreDirsRequest,
  673. tenant: TenantContext = Depends(get_tenant),
  674. db: Database = Depends(get_database),
  675. ):
  676. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  677. if not kb:
  678. raise HTTPException(status_code=404, detail="Knowledge base not found")
  679. # Normalise: strip slashes, dedup, sort
  680. clean = sorted({p.strip("/").strip() for p in req.ignored_dirs if p.strip()})
  681. db.execute(
  682. "UPDATE knowledge_bases SET ignore_dirs=?, updated_at=? WHERE id=?",
  683. (json.dumps(clean), now_utc(), kb_id)
  684. )
  685. db.commit()
  686. return {"ignored_dirs": clean}
  687. @router.post("/{kb_id}/files", status_code=201)
  688. async def add_files(
  689. kb_id: str,
  690. req: AddFilesRequest,
  691. tenant: TenantContext = Depends(get_tenant),
  692. db: Database = Depends(get_database),
  693. ):
  694. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  695. if not kb:
  696. raise HTTPException(status_code=404, detail="Knowledge base not found")
  697. added = []
  698. for fp in req.file_paths:
  699. p = Path(fp)
  700. if not p.is_file():
  701. continue
  702. existing = db.fetchone("SELECT id FROM kb_files WHERE kb_id=? AND file_path=?", (kb_id, str(p)))
  703. if existing:
  704. continue
  705. fid = gen_id("kbf_")
  706. db.execute(
  707. "INSERT INTO kb_files (id, kb_id, file_path, file_name, file_type, size, status, added_at) "
  708. "VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)",
  709. (fid, kb_id, str(p), p.name, p.suffix.lower().lstrip("."), p.stat().st_size, now_utc())
  710. )
  711. added.append(fid)
  712. db.commit()
  713. return {"added": len(added)}
  714. @router.delete("/{kb_id}/files/{file_id}", status_code=204)
  715. async def remove_file(
  716. kb_id: str,
  717. file_id: str,
  718. tenant: TenantContext = Depends(get_tenant),
  719. db: Database = Depends(get_database),
  720. ):
  721. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  722. if not kb:
  723. raise HTTPException(status_code=404, detail="Knowledge base not found")
  724. db.execute("DELETE FROM kb_files WHERE id=? AND kb_id=?", (file_id, kb_id))
  725. db.commit()
  726. return None
  727. # ─── Index Build Jobs ───
  728. @router.post("/{kb_id}/index", status_code=202)
  729. async def trigger_index(
  730. kb_id: str,
  731. req: IndexBuildRequest,
  732. tenant: TenantContext = Depends(get_tenant),
  733. db: Database = Depends(get_database),
  734. ):
  735. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  736. if not kb:
  737. raise HTTPException(status_code=404, detail="Knowledge base not found")
  738. job_id = gen_id("kbj_")
  739. db.execute(
  740. "INSERT INTO kb_jobs (id, kb_id, tenant_id, job_type, status, created_at) VALUES (?,?,?,?,?,?)",
  741. (job_id, kb_id, tenant.tenant_id, f"index:{req.index_type}", "pending", now_utc())
  742. )
  743. db.commit()
  744. t = threading.Thread(
  745. target=_run_index_job,
  746. args=(db, kb_id, job_id, kb["root_dir"], req.index_type, req.mode, req.chunk_strategy),
  747. daemon=True,
  748. )
  749. t.start()
  750. return {"job_id": job_id, "status": "pending", "chunk_strategy": req.chunk_strategy}
  751. # ─── Wiki Compile Jobs ───
  752. @router.post("/{kb_id}/wiki", status_code=202)
  753. async def trigger_wiki(
  754. kb_id: str,
  755. req: WikiBuildRequest,
  756. tenant: TenantContext = Depends(get_tenant),
  757. db: Database = Depends(get_database),
  758. ):
  759. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  760. if not kb:
  761. raise HTTPException(status_code=404, detail="Knowledge base not found")
  762. job_id = gen_id("kbj_")
  763. db.execute(
  764. "INSERT INTO kb_jobs (id, kb_id, tenant_id, job_type, status, created_at) VALUES (?,?,?,?,?,?)",
  765. (job_id, kb_id, tenant.tenant_id, "wiki", "pending", now_utc())
  766. )
  767. db.commit()
  768. t = threading.Thread(
  769. target=_run_wiki_job,
  770. args=(db, kb_id, job_id, kb["root_dir"], req.mode, req.concurrency),
  771. daemon=True,
  772. )
  773. t.start()
  774. return {"job_id": job_id, "status": "pending"}
  775. @router.put("/{kb_id}/wiki/pause", status_code=200)
  776. async def pause_wiki(
  777. kb_id: str,
  778. tenant: TenantContext = Depends(get_tenant),
  779. db: Database = Depends(get_database),
  780. ):
  781. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  782. if not kb:
  783. raise HTTPException(status_code=404, detail="Knowledge base not found")
  784. # Write a stop flag that wiki_compile.py checks
  785. stop_file = Path(kb["root_dir"]) / ".wiki_stop"
  786. if stop_file.exists():
  787. stop_file.unlink()
  788. return {"action": "resumed"}
  789. else:
  790. stop_file.touch()
  791. return {"action": "paused"}
  792. # ─── Wiki Content ───
  793. @router.get("/{kb_id}/wiki/tree")
  794. async def wiki_tree(
  795. kb_id: str,
  796. tenant: TenantContext = Depends(get_tenant),
  797. db: Database = Depends(get_database),
  798. ):
  799. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  800. if not kb:
  801. raise HTTPException(status_code=404, detail="Knowledge base not found")
  802. wiki_path = Path(kb["root_dir"]) / "wiki"
  803. result: Dict[str, Any] = {}
  804. for sub in ["sources", "entities", "topics", "analyses"]:
  805. d = wiki_path / sub
  806. pages = []
  807. if d.is_dir():
  808. for f in sorted(d.glob("*.md")):
  809. s = f.stat()
  810. pages.append({
  811. "name": f.stem,
  812. "path": f"{sub}/{f.name}",
  813. "size": s.st_size,
  814. "modified": s.st_mtime,
  815. })
  816. result[sub] = pages
  817. idx = wiki_path / "index.md"
  818. result["index_content"] = idx.read_text(encoding="utf-8") if idx.exists() else ""
  819. return result
  820. @router.get("/{kb_id}/wiki/schedule")
  821. async def get_wiki_schedule(
  822. kb_id: str,
  823. tenant: TenantContext = Depends(get_tenant),
  824. db: Database = Depends(get_database),
  825. ):
  826. """Return the current auto-recompile schedule for this knowledge base."""
  827. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  828. if not kb:
  829. raise HTTPException(status_code=404, detail="Knowledge base not found")
  830. return _load_wiki_schedule(kb["root_dir"])
  831. @router.put("/{kb_id}/wiki/schedule")
  832. async def update_wiki_schedule(
  833. kb_id: str,
  834. req: WikiScheduleRequest,
  835. tenant: TenantContext = Depends(get_tenant),
  836. db: Database = Depends(get_database),
  837. ):
  838. """Enable / disable the auto-recompile schedule and set the interval."""
  839. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  840. if not kb:
  841. raise HTTPException(status_code=404, detail="Knowledge base not found")
  842. cfg = _load_wiki_schedule(kb["root_dir"])
  843. cfg["enabled"] = req.enabled
  844. cfg["interval"] = req.interval
  845. cfg["next_run"] = _compute_next_run(req.interval) if req.enabled else cfg.get("next_run", "")
  846. _save_wiki_schedule(kb["root_dir"], cfg)
  847. return cfg
  848. @router.get("/{kb_id}/wiki/{page_path:path}")
  849. async def wiki_page(
  850. kb_id: str,
  851. page_path: str,
  852. tenant: TenantContext = Depends(get_tenant),
  853. db: Database = Depends(get_database),
  854. ):
  855. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  856. if not kb:
  857. raise HTTPException(status_code=404, detail="Knowledge base not found")
  858. wiki_path = Path(kb["root_dir"]) / "wiki"
  859. full = (wiki_path / page_path).resolve()
  860. # Path traversal check
  861. if not str(full).startswith(str(wiki_path.resolve())):
  862. raise HTTPException(status_code=403, detail="Forbidden")
  863. if not full.exists():
  864. raise HTTPException(status_code=404, detail="Page not found")
  865. return {"path": page_path, "content": full.read_text(encoding="utf-8"), "size": full.stat().st_size}
  866. @router.delete("/{kb_id}/wiki/analyses/{name}", status_code=204)
  867. async def delete_analysis(
  868. kb_id: str,
  869. name: str,
  870. tenant: TenantContext = Depends(get_tenant),
  871. db: Database = Depends(get_database),
  872. ):
  873. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  874. if not kb:
  875. raise HTTPException(status_code=404, detail="Knowledge base not found")
  876. p = Path(kb["root_dir"]) / "wiki" / "analyses" / f"{name}.md"
  877. if p.exists() and str(p.resolve()).startswith(str((Path(kb["root_dir"]) / "wiki").resolve())):
  878. p.unlink()
  879. return None
  880. # ─── P0 helper: sync LLM call (non-streaming) ───
  881. def _call_llm_once(
  882. chat_url: str,
  883. headers: dict,
  884. model: str,
  885. messages: List[Dict],
  886. is_anthropic: bool = False,
  887. max_tokens: int = 256,
  888. timeout: int = 30,
  889. ) -> str:
  890. """Make a single non-streaming LLM call and return the full response text."""
  891. import urllib.request as _ur
  892. if is_anthropic:
  893. system_parts = [m["content"] for m in messages if m["role"] == "system"]
  894. user_msgs = [m for m in messages if m["role"] != "system"]
  895. body_dict: dict = {
  896. "model": model,
  897. "messages": user_msgs,
  898. "max_tokens": max_tokens,
  899. "stream": False,
  900. }
  901. if system_parts:
  902. body_dict["system"] = "\n\n".join(system_parts)
  903. body = json.dumps(body_dict, ensure_ascii=False).encode("utf-8")
  904. req = _ur.Request(chat_url, data=body, headers=headers, method="POST")
  905. with _ur.urlopen(req, timeout=timeout) as resp:
  906. d = json.loads(resp.read())
  907. return d.get("content", [{}])[0].get("text", "")
  908. else:
  909. body = json.dumps({
  910. "model": model,
  911. "messages": messages,
  912. "max_tokens": max_tokens,
  913. "stream": False,
  914. }, ensure_ascii=False).encode("utf-8")
  915. req = _ur.Request(chat_url, data=body, headers=headers, method="POST")
  916. with _ur.urlopen(req, timeout=timeout) as resp:
  917. d = json.loads(resp.read())
  918. return d["choices"][0]["message"].get("content", "")
  919. # ─── P0 helper: Reciprocal Rank Fusion ───
  920. def _rrf_fuse(all_rankings: List[List[Dict]], k: int = 60) -> List[Dict]:
  921. """Reciprocal Rank Fusion: score(d) += 1/(k + rank) across all rankings.
  922. Returns a deduplicated list sorted by descending RRF score.
  923. Deduplication key: (source, first 80 chars of text).
  924. """
  925. scores: Dict[str, float] = {}
  926. best: Dict[str, Dict] = {}
  927. for ranking in all_rankings:
  928. for rank, chunk in enumerate(ranking, start=1):
  929. key = f"{chunk.get('source', '')}::{(chunk.get('text') or '')[:80]}"
  930. scores[key] = scores.get(key, 0.0) + 1.0 / (k + rank)
  931. if key not in best:
  932. best[key] = chunk
  933. return sorted(best.values(), key=lambda c: -scores[
  934. f"{c.get('source', '')}::{(c.get('text') or '')[:80]}"
  935. ])
  936. # ─── P0 helper: Wiki schedule persistence ───
  937. def _load_wiki_schedule(kb_root: str) -> dict:
  938. """Read {kb_root}/.wiki_schedule.json; return defaults if absent."""
  939. p = Path(kb_root) / ".wiki_schedule.json"
  940. defaults = {"enabled": False, "interval": "weekly", "next_run": "", "last_run": ""}
  941. if not p.exists():
  942. return defaults
  943. try:
  944. return {**defaults, **json.loads(p.read_text(encoding="utf-8"))}
  945. except Exception:
  946. return defaults
  947. def _save_wiki_schedule(kb_root: str, config: dict):
  948. """Write {kb_root}/.wiki_schedule.json atomically."""
  949. p = Path(kb_root) / ".wiki_schedule.json"
  950. tmp = p.with_suffix(".json.tmp")
  951. tmp.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8")
  952. tmp.replace(p)
  953. def _compute_next_run(interval: str) -> str:
  954. """Return ISO-8601 UTC timestamp for the next run based on interval."""
  955. import datetime
  956. now = datetime.datetime.utcnow()
  957. if interval == "daily":
  958. delta = datetime.timedelta(days=1)
  959. else: # weekly
  960. delta = datetime.timedelta(weeks=1)
  961. return (now + delta).strftime("%Y-%m-%dT%H:%M:%SZ")
  962. # ─── P0 helper: Contextual chunking enrichment ───
  963. def _contextual_enrich_index(
  964. kb_path: Path,
  965. scripts_dir: Path,
  966. log_buf: List[str],
  967. db: Database,
  968. job_id: str,
  969. flush_fn,
  970. provider_id: str = "",
  971. model_override: str = "",
  972. ):
  973. """After BM25 build: read rag_index.json, group chunks by source doc, generate one LLM
  974. summary per doc, prepend '[文档背景: {summary}]\\n' to every chunk from that doc, and
  975. write the enriched index back.
  976. Falls back gracefully on any error so the main index job is never broken.
  977. """
  978. index_file = kb_path / "rag_index.json"
  979. if not index_file.exists():
  980. log_buf.append("[Contextual] rag_index.json 不存在,跳过上下文增强")
  981. return
  982. try:
  983. from agentpaas.api.v1.providers_api import DEFAULT_PROVIDERS
  984. chat_url, headers, model_name, _ = _resolve_llm(provider_id, model_override)
  985. pdef = DEFAULT_PROVIDERS.get(provider_id, {})
  986. is_anthropic = pdef.get("type") == "anthropic"
  987. except Exception as e:
  988. log_buf.append(f"[Contextual] LLM 解析失败({e}),跳过上下文增强")
  989. flush_fn("running", log_buf)
  990. return
  991. try:
  992. raw = json.loads(index_file.read_text(encoding="utf-8"))
  993. except Exception as e:
  994. log_buf.append(f"[Contextual] 读取 rag_index.json 失败({e})")
  995. return
  996. # raw can be list of chunks or dict; normalise
  997. chunks: List[Dict] = raw if isinstance(raw, list) else raw.get("chunks", [])
  998. # Group chunks by source document
  999. from collections import defaultdict
  1000. doc_chunks: Dict[str, List[int]] = defaultdict(list)
  1001. for idx, chunk in enumerate(chunks):
  1002. src = chunk.get("source") or chunk.get("file") or ""
  1003. doc_chunks[src].append(idx)
  1004. log_buf.append(f"[Contextual] 共 {len(doc_chunks)} 篇文档,生成摘要中...")
  1005. flush_fn("running", log_buf)
  1006. summaries: Dict[str, str] = {}
  1007. for src, idxs in doc_chunks.items():
  1008. # Gather up to 1500 chars of text as doc sample
  1009. sample = "\n".join((chunks[i].get("text") or "")[:300] for i in idxs[:5])[:1500]
  1010. prompt = [
  1011. {"role": "system", "content": "你是文档摘要助手,请用1-3句话概括以下文档的主要内容。"},
  1012. {"role": "user", "content": f"文档: {src}\n\n内容摘录:\n{sample}\n\n一句话摘要:"},
  1013. ]
  1014. try:
  1015. summary = _call_llm_once(
  1016. chat_url, headers, model_name, prompt,
  1017. is_anthropic=is_anthropic, max_tokens=128, timeout=20,
  1018. ).strip()
  1019. summaries[src] = summary
  1020. except Exception as e:
  1021. summaries[src] = src # fallback: just use filename
  1022. # Prepend background context to each chunk
  1023. enriched = 0
  1024. for chunk in chunks:
  1025. src = chunk.get("source") or chunk.get("file") or ""
  1026. summary = summaries.get(src, "")
  1027. if summary and not (chunk.get("text") or "").startswith("[文档背景:"):
  1028. chunk["text"] = f"[文档背景: {summary}]\n{chunk.get('text', '')}"
  1029. enriched += 1
  1030. # Write back
  1031. try:
  1032. if isinstance(raw, list):
  1033. out = chunks
  1034. else:
  1035. raw["chunks"] = chunks
  1036. out = raw
  1037. index_file.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
  1038. log_buf.append(f"[Contextual] ✅ 已为 {enriched} 个 chunk 添加文档背景({len(summaries)} 篇文档)")
  1039. except Exception as e:
  1040. log_buf.append(f"[Contextual] 写回索引失败({e})")
  1041. flush_fn("running", log_buf)
  1042. # ─── P0 helper: Wiki auto-recompile background scheduler ───
  1043. _SCHEDULE_CHECKER_STARTED = False
  1044. _SCHEDULE_CHECKER_LOCK = threading.Lock()
  1045. def _wiki_schedule_loop():
  1046. """Daemon thread: every 30 min, scan all KB roots and trigger wiki builds if due."""
  1047. import datetime
  1048. from agentpaas.db.session import get_db
  1049. while True:
  1050. time.sleep(1800) # 30 minutes
  1051. try:
  1052. db: Database = get_db()
  1053. kbs = db.fetchall("SELECT id, root_dir FROM knowledge_bases WHERE root_dir IS NOT NULL")
  1054. now_str = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
  1055. for kb in kbs:
  1056. try:
  1057. cfg = _load_wiki_schedule(kb["root_dir"])
  1058. if not cfg.get("enabled"):
  1059. continue
  1060. next_run = cfg.get("next_run", "")
  1061. if not next_run or now_str >= next_run:
  1062. # Trigger wiki build
  1063. job_id = gen_id("kbj_")
  1064. tenant_row = db.fetchone(
  1065. "SELECT tenant_id FROM knowledge_bases WHERE id=?", (kb["id"],)
  1066. )
  1067. tid = tenant_row["tenant_id"] if tenant_row else "system"
  1068. db.execute(
  1069. "INSERT INTO kb_jobs (id, kb_id, tenant_id, job_type, status, created_at) VALUES (?,?,?,?,?,?)",
  1070. (job_id, kb["id"], tid, "wiki:scheduled", "pending", now_utc()),
  1071. )
  1072. db.commit()
  1073. t = threading.Thread(
  1074. target=_run_wiki_job,
  1075. args=(db, kb["id"], job_id, kb["root_dir"], "incremental", 2),
  1076. daemon=True,
  1077. )
  1078. t.start()
  1079. # Update schedule
  1080. cfg["last_run"] = now_str
  1081. cfg["next_run"] = _compute_next_run(cfg.get("interval", "weekly"))
  1082. _save_wiki_schedule(kb["root_dir"], cfg)
  1083. except Exception:
  1084. pass
  1085. except Exception:
  1086. pass
  1087. def _start_schedule_checker():
  1088. """Start the wiki-schedule background thread exactly once."""
  1089. global _SCHEDULE_CHECKER_STARTED
  1090. with _SCHEDULE_CHECKER_LOCK:
  1091. if _SCHEDULE_CHECKER_STARTED:
  1092. return
  1093. _SCHEDULE_CHECKER_STARTED = True
  1094. t = threading.Thread(
  1095. target=_wiki_schedule_loop,
  1096. daemon=True,
  1097. name="wiki-scheduler",
  1098. )
  1099. t.start()
  1100. # ─── P1: PageIndex — no-vector page-level RAG ───
  1101. def _build_page_index(
  1102. kb_path: Path,
  1103. log_buf: List[str],
  1104. db: "Database",
  1105. job_id: str,
  1106. flush_fn,
  1107. provider_id: str = "",
  1108. model_override: str = "",
  1109. ):
  1110. """Build rag_page_index.json from processed/ text files.
  1111. Each page becomes one index entry:
  1112. {id, source, page, summary, text, length}
  1113. Pages are detected by '\\f' (form feed) or '--- Page N ---' markers that
  1114. text extractors like extract_docs.py typically emit. Falls back to
  1115. splitting every ~3000 chars when no page markers are found.
  1116. The summary is generated by LLM (one call per page); falls back to the
  1117. first 150 chars of the page text if LLM is unavailable.
  1118. """
  1119. processed_dir = kb_path / "processed"
  1120. if not processed_dir.is_dir():
  1121. log_buf.append("[PageIndex] processed/ 目录不存在,跳过")
  1122. return
  1123. txt_files = list(processed_dir.glob("*.txt"))
  1124. if not txt_files:
  1125. log_buf.append("[PageIndex] processed/ 中没有 .txt 文件,跳过")
  1126. return
  1127. # Try to get LLM for summaries (graceful fallback if unavailable)
  1128. chat_url = headers = model_name = None
  1129. is_anthropic = False
  1130. try:
  1131. from agentpaas.api.v1.providers_api import DEFAULT_PROVIDERS
  1132. chat_url, headers, model_name, _ = _resolve_llm(provider_id, model_override)
  1133. pdef = DEFAULT_PROVIDERS.get(provider_id, {})
  1134. is_anthropic = pdef.get("type") == "anthropic"
  1135. except Exception:
  1136. pass # No LLM → use truncated text as summary
  1137. log_buf.append(f"[PageIndex] 处理 {len(txt_files)} 个文档,生成页面索引...")
  1138. flush_fn("running", log_buf)
  1139. entries: List[Dict] = []
  1140. for txt_file in sorted(txt_files):
  1141. source = txt_file.stem
  1142. try:
  1143. text = txt_file.read_text(encoding="utf-8", errors="replace")
  1144. except Exception:
  1145. continue
  1146. # Split into pages
  1147. pages: List[str] = []
  1148. if "\f" in text:
  1149. # Form-feed page breaks (PDF extractors)
  1150. pages = [p.strip() for p in text.split("\f") if p.strip()]
  1151. else:
  1152. import re
  1153. # "--- Page N ---" markers
  1154. parts = re.split(r"---\s*[Pp]age\s*\d+\s*---", text)
  1155. if len(parts) > 1:
  1156. pages = [p.strip() for p in parts if p.strip()]
  1157. else:
  1158. # No markers: split every ~3000 chars at paragraph boundary
  1159. chunk_size = 3000
  1160. remaining = text
  1161. while remaining:
  1162. if len(remaining) <= chunk_size:
  1163. pages.append(remaining.strip())
  1164. break
  1165. split_at = remaining.rfind("\n\n", 0, chunk_size)
  1166. if split_at == -1:
  1167. split_at = chunk_size
  1168. pages.append(remaining[:split_at].strip())
  1169. remaining = remaining[split_at:].strip()
  1170. for page_num, page_text in enumerate(pages, start=1):
  1171. if not page_text:
  1172. continue
  1173. # Generate summary
  1174. summary = page_text[:150] # default fallback
  1175. if chat_url:
  1176. try:
  1177. prompt = [
  1178. {"role": "system", "content": "请用1-2句话概括以下页面内容,突出关键信息。"},
  1179. {"role": "user", "content": page_text[:1500]},
  1180. ]
  1181. summary = _call_llm_once(
  1182. chat_url, headers, model_name, prompt,
  1183. is_anthropic=is_anthropic, max_tokens=100, timeout=15,
  1184. ).strip() or page_text[:150]
  1185. except Exception:
  1186. pass
  1187. entries.append({
  1188. "id": f"{source}::p{page_num}",
  1189. "source": source,
  1190. "page": page_num,
  1191. "summary": summary,
  1192. "text": page_text,
  1193. "length": len(page_text),
  1194. })
  1195. if entries:
  1196. index_file = kb_path / "rag_page_index.json"
  1197. index_file.write_text(json.dumps(entries, ensure_ascii=False, indent=2), encoding="utf-8")
  1198. log_buf.append(f"[PageIndex] ✅ 已生成 {len(entries)} 个页面条目({len(txt_files)} 篇文档)→ rag_page_index.json")
  1199. else:
  1200. log_buf.append("[PageIndex] ⚠️ 未生成任何页面条目")
  1201. flush_fn("running", log_buf)
  1202. def _page_index_search(kb_root: str, query: str, top_k: int) -> List[Dict]:
  1203. """BM25-style search over rag_page_index.json (summaries + text).
  1204. Uses a simple TF token overlap score since we avoid importing rank_bm25
  1205. at the top level. Returns results in {id, source, page, summary, text, score} format.
  1206. """
  1207. index_file = Path(kb_root) / "rag_page_index.json"
  1208. if not index_file.exists():
  1209. return []
  1210. try:
  1211. entries = json.loads(index_file.read_text(encoding="utf-8"))
  1212. except Exception:
  1213. return []
  1214. # Tokenise query
  1215. import re
  1216. query_tokens = set(re.sub(r"\W+", " ", query.lower()).split())
  1217. if not query_tokens:
  1218. return entries[:top_k]
  1219. scored: List[tuple] = []
  1220. for entry in entries:
  1221. haystack = f"{entry.get('summary','')} {entry.get('text','')}".lower()
  1222. score = sum(1 for tok in query_tokens if tok in haystack) / max(len(query_tokens), 1)
  1223. scored.append((score, entry))
  1224. scored.sort(key=lambda x: -x[0])
  1225. results = []
  1226. for score, entry in scored[:top_k]:
  1227. results.append({**entry, "score": round(score, 4)})
  1228. return results
  1229. # ─── Search helpers ───
  1230. def _search_subprocess(kb_root: str, scripts_dir: Path, mode: str, query: str, top_k: int) -> List[Dict]:
  1231. """Run search in an isolated subprocess to avoid config singleton / module cache issues.
  1232. search_engine.py uses `from config import cfg` at module level. Once imported
  1233. in-process the singleton is frozen to the first kb_root it saw. Running as a
  1234. subprocess gives each search a fresh Python interpreter with the correct
  1235. INSTANCE_CONFIG / KNOWLEDGE_DIR env vars.
  1236. """
  1237. instance_yml = _write_instance_yml(kb_root)
  1238. try:
  1239. env = _make_script_env(kb_root, scripts_dir, {"INSTANCE_CONFIG": instance_yml})
  1240. # Build the inline runner script — use json.dumps for safe string embedding
  1241. import_stmt = (
  1242. "from search_engine import search"
  1243. if mode == "bm25"
  1244. else (
  1245. "try:\n from search_unified import search\n"
  1246. "except ImportError:\n from search_engine import search"
  1247. )
  1248. )
  1249. runner = (
  1250. f"import sys, os, json\n"
  1251. f"sys.path.insert(0, {json.dumps(str(scripts_dir))})\n"
  1252. f"os.environ['INSTANCE_CONFIG'] = {json.dumps(instance_yml)}\n"
  1253. f"os.environ['KNOWLEDGE_DIR'] = {json.dumps(kb_root)}\n"
  1254. f"{import_stmt}\n"
  1255. f"results = search({json.dumps(query)}, top_k={top_k})\n"
  1256. f"print(json.dumps(results, ensure_ascii=False))\n"
  1257. )
  1258. r = subprocess.run(
  1259. [sys.executable, "-c", runner],
  1260. capture_output=True, text=True, timeout=30, env=env,
  1261. )
  1262. if r.returncode != 0:
  1263. raise RuntimeError(r.stderr[-800:] or "搜索子进程异常退出")
  1264. # Last line of stdout is the JSON result (earlier lines may be progress logs)
  1265. lines = [l for l in r.stdout.strip().splitlines() if l.strip()]
  1266. if not lines:
  1267. return []
  1268. return json.loads(lines[-1])
  1269. finally:
  1270. try:
  1271. os.unlink(instance_yml)
  1272. except Exception:
  1273. pass
  1274. def _fusion_search(
  1275. kb_root: str,
  1276. scripts_dir: Path,
  1277. query: str,
  1278. top_k: int,
  1279. n_subqueries: int = 4,
  1280. provider_id: str = "",
  1281. model_override: str = "",
  1282. ) -> List[Dict]:
  1283. """RAG Fusion: expand query → N sub-queries → BM25 search each → RRF merge.
  1284. If LLM expansion fails, falls back to a single BM25 search on the original query.
  1285. """
  1286. # ── Step 1: query expansion ──
  1287. sub_queries: List[str] = [query]
  1288. try:
  1289. from agentpaas.api.v1.providers_api import DEFAULT_PROVIDERS
  1290. chat_url, headers, model_name, _ = _resolve_llm(provider_id, model_override)
  1291. pdef = DEFAULT_PROVIDERS.get(provider_id, {})
  1292. is_anthropic = pdef.get("type") == "anthropic"
  1293. expansion_prompt = [
  1294. {
  1295. "role": "system",
  1296. "content": (
  1297. f"请将用户的问题改写为 {n_subqueries} 个语义相近但措辞不同的子问题,"
  1298. "每行一个,只输出问题,不要编号或其他内容。"
  1299. ),
  1300. },
  1301. {"role": "user", "content": query},
  1302. ]
  1303. raw = _call_llm_once(
  1304. chat_url, headers, model_name, expansion_prompt,
  1305. is_anthropic=is_anthropic, max_tokens=256, timeout=20,
  1306. )
  1307. lines = [l.strip() for l in raw.splitlines() if l.strip()]
  1308. if lines:
  1309. sub_queries = [query] + lines[:n_subqueries]
  1310. except Exception:
  1311. pass # fallback: single query
  1312. # ── Step 2: search each sub-query ──
  1313. all_rankings: List[List[Dict]] = []
  1314. for sq in sub_queries:
  1315. try:
  1316. results = _search_subprocess(kb_root, scripts_dir, "bm25", sq, top_k)
  1317. if results:
  1318. all_rankings.append(results)
  1319. except Exception:
  1320. pass
  1321. if not all_rankings:
  1322. # Last resort: direct BM25
  1323. try:
  1324. return _search_subprocess(kb_root, scripts_dir, "bm25", query, top_k)
  1325. except Exception:
  1326. return []
  1327. # ── Step 3: RRF fusion ──
  1328. return _rrf_fuse(all_rankings)[:top_k]
  1329. # ─── Search ───
  1330. @router.post("/{kb_id}/search")
  1331. async def search_kb(
  1332. kb_id: str,
  1333. req: SearchRequest,
  1334. tenant: TenantContext = Depends(get_tenant),
  1335. db: Database = Depends(get_database),
  1336. ):
  1337. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  1338. if not kb:
  1339. raise HTTPException(status_code=404, detail="Knowledge base not found")
  1340. kb_root = kb["root_dir"]
  1341. scripts_dir = _find_scripts_dir(Path(kb_root))
  1342. if not scripts_dir:
  1343. raise HTTPException(status_code=503, detail="搜索脚本未找到,请先构建 BM25 索引。")
  1344. # Check index exists
  1345. index_file = Path(kb_root) / "rag_index.json"
  1346. if not index_file.exists():
  1347. raise HTTPException(status_code=503, detail="BM25 索引不存在,请先在「索引构建」标签中触发构建。")
  1348. try:
  1349. if req.mode == "fusion":
  1350. results = await asyncio.get_event_loop().run_in_executor(
  1351. None,
  1352. lambda: _fusion_search(kb_root, scripts_dir, req.query, req.top_k),
  1353. )
  1354. elif req.mode == "pageindex":
  1355. results = await asyncio.get_event_loop().run_in_executor(
  1356. None,
  1357. lambda: _page_index_search(kb_root, req.query, req.top_k),
  1358. )
  1359. else:
  1360. results = await asyncio.get_event_loop().run_in_executor(
  1361. None,
  1362. lambda: _search_subprocess(kb_root, scripts_dir, req.mode, req.query, req.top_k)
  1363. )
  1364. return {
  1365. "query": req.query,
  1366. "mode": req.mode,
  1367. "results": results[:req.top_k],
  1368. }
  1369. except RuntimeError as e:
  1370. raise HTTPException(status_code=500, detail=str(e))
  1371. except Exception as e:
  1372. raise HTTPException(status_code=500, detail=str(e))
  1373. def _detect_ollama_model() -> tuple[str, str]:
  1374. """Return (base_url, model_name) for the best available local LLM.
  1375. Probes Ollama at 11434/11435, picks qwen2.5:7b if available, else first model found.
  1376. """
  1377. import urllib.request as _ur
  1378. preferred = ["qwen2.5:7b", "qwen2.5:14b", "qwen2.5:32b", "qwen2.5-coder:7b"]
  1379. for port in (11434, 11435):
  1380. base = f"http://127.0.0.1:{port}"
  1381. try:
  1382. with _ur.urlopen(f"{base}/api/tags", timeout=2) as r:
  1383. data = json.loads(r.read())
  1384. models = [m["name"] for m in data.get("models", [])]
  1385. if not models:
  1386. continue
  1387. chosen = next((m for m in preferred if m in models), models[0])
  1388. return base, chosen
  1389. except Exception:
  1390. continue
  1391. return "http://127.0.0.1:11434", "qwen2.5:7b"
  1392. def _resolve_llm(provider_id: str, model: str) -> tuple[str, dict, str, str]:
  1393. """Resolve LLM call parameters from a configured provider.
  1394. Returns (chat_url, headers, model_name, display_label).
  1395. Falls back to local Ollama if provider_id is empty or "ollama".
  1396. """
  1397. from agentpaas.api.v1.providers_api import DEFAULT_PROVIDERS, _load_providers
  1398. # ── Ollama fallback ──────────────────────────────────────────
  1399. if not provider_id or provider_id == "ollama":
  1400. base, auto_model = _detect_ollama_model()
  1401. chosen = model or auto_model
  1402. chat_url = f"{base}/v1/chat/completions"
  1403. return chat_url, {"Content-Type": "application/json"}, chosen, f"Ollama / {chosen}"
  1404. pdef = DEFAULT_PROVIDERS.get(provider_id)
  1405. if not pdef:
  1406. raise ValueError(f"未知提供商: {provider_id}")
  1407. saved = _load_providers()
  1408. entry = saved.get(provider_id, {})
  1409. base_url = (entry.get("base_url") or pdef.get("base_url", "")).rstrip("/")
  1410. api_key = entry.get("api_key") or os.getenv(pdef.get("env_key", ""), "")
  1411. chosen = model or (pdef["models"][0] if pdef.get("models") else "")
  1412. ptype = pdef.get("type", "openai_compatible")
  1413. # ── Ollama type ──────────────────────────────────────────────
  1414. if ptype == "ollama":
  1415. # Detect actual running port from saved/default base_url
  1416. # but try to also auto-probe in case port changed
  1417. try:
  1418. auto_base, auto_model = _detect_ollama_model()
  1419. except Exception:
  1420. auto_base = base_url
  1421. auto_model = chosen
  1422. final_base = entry.get("base_url") or auto_base
  1423. chosen = model or auto_model
  1424. chat_url = f"{final_base.rstrip('/')}/v1/chat/completions"
  1425. return chat_url, {"Content-Type": "application/json"}, chosen, f"Ollama / {chosen}"
  1426. # ── claude-code type (not streamable via HTTP) ────────────────
  1427. if ptype == "claude-code":
  1428. raise ValueError("claude-code 提供商不支持 HTTP 流式调用,请选择其他提供商")
  1429. # ── Anthropic native API ──────────────────────────────────────
  1430. if ptype == "anthropic":
  1431. if not api_key:
  1432. raise ValueError("Anthropic API Key 未配置")
  1433. chat_url = f"{base_url}/v1/messages"
  1434. headers = {
  1435. "Content-Type": "application/json",
  1436. "x-api-key": api_key,
  1437. "anthropic-version": "2023-06-01",
  1438. }
  1439. return chat_url, headers, chosen, f"Anthropic / {chosen}"
  1440. # ── OpenAI / OpenAI-compatible ────────────────────────────────
  1441. if not api_key:
  1442. raise ValueError(f"提供商 {pdef['name']} 未配置 API Key")
  1443. chat_url = f"{base_url}/chat/completions"
  1444. headers = {
  1445. "Content-Type": "application/json",
  1446. "Authorization": f"Bearer {api_key}",
  1447. }
  1448. return chat_url, headers, chosen, f"{pdef['name']} / {chosen}"
  1449. def _stream_openai_compat(
  1450. chat_url: str, headers: dict, model: str, messages: list, temperature: float = 0.3
  1451. ):
  1452. """Generator that yields text delta strings from an OpenAI-compatible streaming endpoint."""
  1453. import urllib.request as _ur
  1454. body = json.dumps({
  1455. "model": model,
  1456. "messages": messages,
  1457. "stream": True,
  1458. "temperature": temperature,
  1459. }, ensure_ascii=False).encode("utf-8")
  1460. req = _ur.Request(chat_url, data=body, headers=headers, method="POST")
  1461. with _ur.urlopen(req, timeout=180) as resp:
  1462. for raw_line in resp:
  1463. line = raw_line.decode("utf-8").strip()
  1464. if not line.startswith("data:"):
  1465. continue
  1466. payload = line[5:].strip()
  1467. if payload == "[DONE]":
  1468. break
  1469. try:
  1470. d = json.loads(payload)
  1471. delta = d["choices"][0]["delta"].get("content", "")
  1472. if delta:
  1473. yield delta
  1474. except Exception:
  1475. continue
  1476. def _stream_anthropic(
  1477. chat_url: str, headers: dict, model: str, messages: list, temperature: float = 0.3
  1478. ):
  1479. """Generator that yields text delta strings from Anthropic Messages streaming API."""
  1480. import urllib.request as _ur
  1481. # Convert OpenAI-style messages to Anthropic format
  1482. system_parts = [m["content"] for m in messages if m["role"] == "system"]
  1483. user_msgs = [m for m in messages if m["role"] != "system"]
  1484. body_dict: dict = {
  1485. "model": model,
  1486. "messages": user_msgs,
  1487. "max_tokens": 2048,
  1488. "stream": True,
  1489. "temperature": temperature,
  1490. }
  1491. if system_parts:
  1492. body_dict["system"] = "\n\n".join(system_parts)
  1493. body = json.dumps(body_dict, ensure_ascii=False).encode("utf-8")
  1494. req = _ur.Request(chat_url, data=body, headers=headers, method="POST")
  1495. with _ur.urlopen(req, timeout=180) as resp:
  1496. for raw_line in resp:
  1497. line = raw_line.decode("utf-8").strip()
  1498. if not line.startswith("data:"):
  1499. continue
  1500. payload = line[5:].strip()
  1501. try:
  1502. d = json.loads(payload)
  1503. if d.get("type") == "content_block_delta":
  1504. delta = d.get("delta", {}).get("text", "")
  1505. if delta:
  1506. yield delta
  1507. except Exception:
  1508. continue
  1509. @router.get("/{kb_id}/search/stream")
  1510. async def search_stream(
  1511. kb_id: str,
  1512. q: str = Query(..., description="Question to answer"),
  1513. mode: str = Query(default="bm25"),
  1514. top_k: int = Query(default=5, ge=1, le=20),
  1515. provider_id: str = Query(default="", description="Provider ID from providers config"),
  1516. model: str = Query(default="", description="Model name override"),
  1517. tenant: TenantContext = Depends(get_tenant),
  1518. db: Database = Depends(get_database),
  1519. ):
  1520. """SSE streaming QA: retrieve → build prompt → stream LLM answer token by token."""
  1521. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  1522. if not kb:
  1523. raise HTTPException(status_code=404, detail="Knowledge base not found")
  1524. kb_root = kb["root_dir"]
  1525. scripts_dir = _find_scripts_dir(Path(kb_root))
  1526. async def event_stream():
  1527. # ── Step 1: retrieve ──
  1528. if mode == "fusion":
  1529. yield f"data: {json.dumps({'type': 'status', 'text': '正在扩写查询并多路检索(RAG Fusion)...'})}\n\n"
  1530. else:
  1531. yield f"data: {json.dumps({'type': 'status', 'text': '正在检索相关文档...'})}\n\n"
  1532. await asyncio.sleep(0)
  1533. chunks: List[Dict] = []
  1534. index_exists = (Path(kb_root) / "rag_index.json").exists()
  1535. page_index_exists = (Path(kb_root) / "rag_page_index.json").exists()
  1536. if mode == "pageindex" and page_index_exists:
  1537. try:
  1538. chunks = await asyncio.get_event_loop().run_in_executor(
  1539. None,
  1540. lambda: _page_index_search(kb_root, q, top_k)
  1541. )
  1542. except Exception as e:
  1543. yield f"data: {json.dumps({'type': 'status', 'text': f'PageIndex 检索出错: {e}'})}\n\n"
  1544. elif scripts_dir and index_exists:
  1545. try:
  1546. if mode == "fusion":
  1547. chunks = await asyncio.get_event_loop().run_in_executor(
  1548. None,
  1549. lambda: _fusion_search(kb_root, scripts_dir, q, top_k, provider_id=provider_id, model_override=model)
  1550. )
  1551. else:
  1552. chunks = await asyncio.get_event_loop().run_in_executor(
  1553. None,
  1554. lambda: _search_subprocess(kb_root, scripts_dir, mode, q, top_k)
  1555. )
  1556. except Exception as e:
  1557. yield f"data: {json.dumps({'type': 'status', 'text': f'检索出错: {e}'})}\n\n"
  1558. else:
  1559. msg = "PageIndex 不存在,请先以 page 分块策略构建索引" if mode == "pageindex" else "BM25 索引不存在,请先构建索引"
  1560. yield f"data: {json.dumps({'type': 'status', 'text': msg})}\n\n"
  1561. # Emit retrieved chunks so the UI can show sources panel
  1562. yield f"data: {json.dumps({'type': 'chunks', 'chunks': chunks[:top_k]})}\n\n"
  1563. await asyncio.sleep(0)
  1564. if not chunks:
  1565. yield f"data: {json.dumps({'type': 'chunk', 'text': '未检索到相关文档,无法生成回答。'})}\n\n"
  1566. yield f"data: {json.dumps({'type': 'done'})}\n\n"
  1567. return
  1568. # ── Step 2: build prompt ──
  1569. context_parts = []
  1570. for i, c in enumerate(chunks[:top_k]):
  1571. context_parts.append(f"[来源{i+1}: {c.get('source', '?')}]\n{c.get('text', '')}")
  1572. context = "\n\n---\n\n".join(context_parts)[:6000]
  1573. messages = [
  1574. {
  1575. "role": "system",
  1576. "content": (
  1577. "你是一个知识库问答助手。请严格基于以下检索到的参考资料回答问题,"
  1578. "若资料中无相关信息请如实说明,回答时用「[来源N]」标注依据。"
  1579. ),
  1580. },
  1581. {
  1582. "role": "user",
  1583. "content": f"## 参考资料\n{context}\n\n## 问题\n{q}\n\n## 回答",
  1584. },
  1585. ]
  1586. # ── Step 3: resolve provider + stream ──
  1587. yield f"data: {json.dumps({'type': 'status', 'text': '正在生成回答...'})}\n\n"
  1588. await asyncio.sleep(0)
  1589. try:
  1590. chat_url, llm_headers, model_name, display_label = _resolve_llm(provider_id, model)
  1591. except Exception as e:
  1592. err_msg = f"提供商配置错误: {e}"
  1593. yield f"data: {json.dumps({'type': 'chunk', 'text': err_msg})}\n\n"
  1594. yield f"data: {json.dumps({'type': 'done'})}\n\n"
  1595. return
  1596. yield f"data: {json.dumps({'type': 'status', 'text': f'模型: {display_label}'})}\n\n"
  1597. await asyncio.sleep(0)
  1598. # Determine stream generator based on provider type
  1599. from agentpaas.api.v1.providers_api import DEFAULT_PROVIDERS
  1600. pdef = DEFAULT_PROVIDERS.get(provider_id, {})
  1601. is_anthropic = pdef.get("type") == "anthropic"
  1602. def _do_stream():
  1603. if is_anthropic:
  1604. yield from _stream_anthropic(chat_url, llm_headers, model_name, messages)
  1605. else:
  1606. yield from _stream_openai_compat(chat_url, llm_headers, model_name, messages)
  1607. try:
  1608. loop = asyncio.get_event_loop()
  1609. # Use asyncio.Queue so the thread can push tokens without blocking the event loop
  1610. # FIX-09: bounded queue prevents OOM from slow clients
  1611. aio_q: asyncio.Queue = asyncio.Queue(maxsize=1000)
  1612. def _fill_q():
  1613. """Runs in a thread — pushes (kind, val) into the asyncio queue."""
  1614. try:
  1615. for delta in _do_stream():
  1616. loop.call_soon_threadsafe(aio_q.put_nowait, ("chunk", delta))
  1617. loop.call_soon_threadsafe(aio_q.put_nowait, ("done", None))
  1618. except Exception as exc:
  1619. loop.call_soon_threadsafe(aio_q.put_nowait, ("error", str(exc)))
  1620. # Start the blocking stream in a thread
  1621. stream_fut = loop.run_in_executor(None, _fill_q)
  1622. # Drain the async queue in the event loop
  1623. while True:
  1624. kind, val = await aio_q.get()
  1625. if kind == "chunk":
  1626. yield f"data: {json.dumps({'type': 'chunk', 'text': val})}\n\n"
  1627. elif kind == "error":
  1628. err_msg = f"\n\n(LLM 错误: {val})"
  1629. yield f"data: {json.dumps({'type': 'chunk', 'text': err_msg})}\n\n"
  1630. break
  1631. else: # done
  1632. break
  1633. await stream_fut # surface thread exceptions
  1634. except Exception as e:
  1635. err_msg = f"\n\n(LLM 错误: {e})"
  1636. yield f"data: {json.dumps({'type': 'chunk', 'text': err_msg})}\n\n"
  1637. yield f"data: {json.dumps({'type': 'done'})}\n\n"
  1638. return StreamingResponse(
  1639. event_stream(),
  1640. media_type="text/event-stream",
  1641. headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
  1642. )
  1643. # ─── Jobs ───
  1644. @router.get("/{kb_id}/jobs")
  1645. async def list_jobs(
  1646. kb_id: str,
  1647. tenant: TenantContext = Depends(get_tenant),
  1648. db: Database = Depends(get_database),
  1649. ):
  1650. kb = db.fetchone("SELECT * FROM knowledge_bases WHERE id=? AND tenant_id=?", (kb_id, tenant.tenant_id))
  1651. if not kb:
  1652. raise HTTPException(status_code=404, detail="Knowledge base not found")
  1653. jobs = db.fetchall(
  1654. "SELECT * FROM kb_jobs WHERE kb_id=? ORDER BY created_at DESC LIMIT 20",
  1655. (kb_id,)
  1656. )
  1657. return {"jobs": jobs}
  1658. @router.get("/{kb_id}/jobs/{job_id}")
  1659. async def get_job(
  1660. kb_id: str,
  1661. job_id: str,
  1662. tenant: TenantContext = Depends(get_tenant),
  1663. db: Database = Depends(get_database),
  1664. ):
  1665. job = db.fetchone("SELECT * FROM kb_jobs WHERE id=? AND kb_id=?", (job_id, kb_id))
  1666. if not job:
  1667. raise HTTPException(status_code=404, detail="Job not found")
  1668. return job