|
@@ -59,7 +59,7 @@ class AddFilesRequest(BaseModel):
|
|
|
class IndexBuildRequest(BaseModel):
|
|
class IndexBuildRequest(BaseModel):
|
|
|
index_type: str = Field(default="bm25", description="bm25 | vector | graph | all")
|
|
index_type: str = Field(default="bm25", description="bm25 | vector | graph | all")
|
|
|
mode: str = Field(default="full", description="full | incremental")
|
|
mode: str = Field(default="full", description="full | incremental")
|
|
|
- chunk_strategy: str = Field(default="token", description="token | contextual")
|
|
|
|
|
|
|
+ chunk_strategy: str = Field(default="token", description="token | contextual | page")
|
|
|
|
|
|
|
|
|
|
|
|
|
class WikiBuildRequest(BaseModel):
|
|
class WikiBuildRequest(BaseModel):
|
|
@@ -69,7 +69,7 @@ class WikiBuildRequest(BaseModel):
|
|
|
|
|
|
|
|
class SearchRequest(BaseModel):
|
|
class SearchRequest(BaseModel):
|
|
|
query: str = Field(..., max_length=4096)
|
|
query: str = Field(..., max_length=4096)
|
|
|
- mode: str = Field(default="lambda", description="lambda | bm25 | vector | wiki | fusion")
|
|
|
|
|
|
|
+ mode: str = Field(default="lambda", description="lambda | bm25 | vector | wiki | fusion | pageindex")
|
|
|
top_k: int = Field(default=5, ge=1, le=20)
|
|
top_k: int = Field(default=5, ge=1, le=20)
|
|
|
weights: Optional[Dict[str, float]] = None # bm25/vector/graph/wiki overrides
|
|
weights: Optional[Dict[str, float]] = None # bm25/vector/graph/wiki overrides
|
|
|
|
|
|
|
@@ -344,6 +344,11 @@ def _run_index_job(db: Database, kb_id: str, job_id: str, kb_root: str, index_ty
|
|
|
log_buf.append("[Contextual] 启动上下文增强分块...")
|
|
log_buf.append("[Contextual] 启动上下文增强分块...")
|
|
|
_flush("running", log_buf)
|
|
_flush("running", log_buf)
|
|
|
_contextual_enrich_index(kb_path, scripts_dir, log_buf, db, job_id, _flush)
|
|
_contextual_enrich_index(kb_path, scripts_dir, log_buf, db, job_id, _flush)
|
|
|
|
|
+ elif chunk_strategy == "page":
|
|
|
|
|
+ # ── Step 2c: PageIndex build (page-level, no vector) ──
|
|
|
|
|
+ log_buf.append("[PageIndex] 构建页面级索引...")
|
|
|
|
|
+ _flush("running", log_buf)
|
|
|
|
|
+ _build_page_index(kb_path, log_buf, db, job_id, _flush)
|
|
|
else:
|
|
else:
|
|
|
log_buf.append("[BM25] rebuild_index.py / build_index.py 均未找到 — 已跳过")
|
|
log_buf.append("[BM25] rebuild_index.py / build_index.py 均未找到 — 已跳过")
|
|
|
|
|
|
|
@@ -1274,6 +1279,155 @@ def _start_schedule_checker():
|
|
|
t.start()
|
|
t.start()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+# ─── P1: PageIndex — no-vector page-level RAG ───
|
|
|
|
|
+
|
|
|
|
|
+def _build_page_index(
|
|
|
|
|
+ kb_path: Path,
|
|
|
|
|
+ log_buf: List[str],
|
|
|
|
|
+ db: "Database",
|
|
|
|
|
+ job_id: str,
|
|
|
|
|
+ flush_fn,
|
|
|
|
|
+ provider_id: str = "",
|
|
|
|
|
+ model_override: str = "",
|
|
|
|
|
+):
|
|
|
|
|
+ """Build rag_page_index.json from processed/ text files.
|
|
|
|
|
+
|
|
|
|
|
+ Each page becomes one index entry:
|
|
|
|
|
+ {id, source, page, summary, text, length}
|
|
|
|
|
+
|
|
|
|
|
+ Pages are detected by '\\f' (form feed) or '--- Page N ---' markers that
|
|
|
|
|
+ text extractors like extract_docs.py typically emit. Falls back to
|
|
|
|
|
+ splitting every ~3000 chars when no page markers are found.
|
|
|
|
|
+
|
|
|
|
|
+ The summary is generated by LLM (one call per page); falls back to the
|
|
|
|
|
+ first 150 chars of the page text if LLM is unavailable.
|
|
|
|
|
+ """
|
|
|
|
|
+ processed_dir = kb_path / "processed"
|
|
|
|
|
+ if not processed_dir.is_dir():
|
|
|
|
|
+ log_buf.append("[PageIndex] processed/ 目录不存在,跳过")
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+ txt_files = list(processed_dir.glob("*.txt"))
|
|
|
|
|
+ if not txt_files:
|
|
|
|
|
+ log_buf.append("[PageIndex] processed/ 中没有 .txt 文件,跳过")
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+ # Try to get LLM for summaries (graceful fallback if unavailable)
|
|
|
|
|
+ chat_url = headers = model_name = None
|
|
|
|
|
+ is_anthropic = False
|
|
|
|
|
+ try:
|
|
|
|
|
+ from agentpaas.api.v1.providers_api import DEFAULT_PROVIDERS
|
|
|
|
|
+ chat_url, headers, model_name, _ = _resolve_llm(provider_id, model_override)
|
|
|
|
|
+ pdef = DEFAULT_PROVIDERS.get(provider_id, {})
|
|
|
|
|
+ is_anthropic = pdef.get("type") == "anthropic"
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass # No LLM → use truncated text as summary
|
|
|
|
|
+
|
|
|
|
|
+ log_buf.append(f"[PageIndex] 处理 {len(txt_files)} 个文档,生成页面索引...")
|
|
|
|
|
+ flush_fn("running", log_buf)
|
|
|
|
|
+
|
|
|
|
|
+ entries: List[Dict] = []
|
|
|
|
|
+ for txt_file in sorted(txt_files):
|
|
|
|
|
+ source = txt_file.stem
|
|
|
|
|
+ try:
|
|
|
|
|
+ text = txt_file.read_text(encoding="utf-8", errors="replace")
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ continue
|
|
|
|
|
+
|
|
|
|
|
+ # Split into pages
|
|
|
|
|
+ pages: List[str] = []
|
|
|
|
|
+ if "\f" in text:
|
|
|
|
|
+ # Form-feed page breaks (PDF extractors)
|
|
|
|
|
+ pages = [p.strip() for p in text.split("\f") if p.strip()]
|
|
|
|
|
+ else:
|
|
|
|
|
+ import re
|
|
|
|
|
+ # "--- Page N ---" markers
|
|
|
|
|
+ parts = re.split(r"---\s*[Pp]age\s*\d+\s*---", text)
|
|
|
|
|
+ if len(parts) > 1:
|
|
|
|
|
+ pages = [p.strip() for p in parts if p.strip()]
|
|
|
|
|
+ else:
|
|
|
|
|
+ # No markers: split every ~3000 chars at paragraph boundary
|
|
|
|
|
+ chunk_size = 3000
|
|
|
|
|
+ remaining = text
|
|
|
|
|
+ while remaining:
|
|
|
|
|
+ if len(remaining) <= chunk_size:
|
|
|
|
|
+ pages.append(remaining.strip())
|
|
|
|
|
+ break
|
|
|
|
|
+ split_at = remaining.rfind("\n\n", 0, chunk_size)
|
|
|
|
|
+ if split_at == -1:
|
|
|
|
|
+ split_at = chunk_size
|
|
|
|
|
+ pages.append(remaining[:split_at].strip())
|
|
|
|
|
+ remaining = remaining[split_at:].strip()
|
|
|
|
|
+
|
|
|
|
|
+ for page_num, page_text in enumerate(pages, start=1):
|
|
|
|
|
+ if not page_text:
|
|
|
|
|
+ continue
|
|
|
|
|
+ # Generate summary
|
|
|
|
|
+ summary = page_text[:150] # default fallback
|
|
|
|
|
+ if chat_url:
|
|
|
|
|
+ try:
|
|
|
|
|
+ prompt = [
|
|
|
|
|
+ {"role": "system", "content": "请用1-2句话概括以下页面内容,突出关键信息。"},
|
|
|
|
|
+ {"role": "user", "content": page_text[:1500]},
|
|
|
|
|
+ ]
|
|
|
|
|
+ summary = _call_llm_once(
|
|
|
|
|
+ chat_url, headers, model_name, prompt,
|
|
|
|
|
+ is_anthropic=is_anthropic, max_tokens=100, timeout=15,
|
|
|
|
|
+ ).strip() or page_text[:150]
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+
|
|
|
|
|
+ entries.append({
|
|
|
|
|
+ "id": f"{source}::p{page_num}",
|
|
|
|
|
+ "source": source,
|
|
|
|
|
+ "page": page_num,
|
|
|
|
|
+ "summary": summary,
|
|
|
|
|
+ "text": page_text,
|
|
|
|
|
+ "length": len(page_text),
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ if entries:
|
|
|
|
|
+ index_file = kb_path / "rag_page_index.json"
|
|
|
|
|
+ index_file.write_text(json.dumps(entries, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
|
+ log_buf.append(f"[PageIndex] ✅ 已生成 {len(entries)} 个页面条目({len(txt_files)} 篇文档)→ rag_page_index.json")
|
|
|
|
|
+ else:
|
|
|
|
|
+ log_buf.append("[PageIndex] ⚠️ 未生成任何页面条目")
|
|
|
|
|
+ flush_fn("running", log_buf)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _page_index_search(kb_root: str, query: str, top_k: int) -> List[Dict]:
|
|
|
|
|
+ """BM25-style search over rag_page_index.json (summaries + text).
|
|
|
|
|
+
|
|
|
|
|
+ Uses a simple TF token overlap score since we avoid importing rank_bm25
|
|
|
|
|
+ at the top level. Returns results in {id, source, page, summary, text, score} format.
|
|
|
|
|
+ """
|
|
|
|
|
+ index_file = Path(kb_root) / "rag_page_index.json"
|
|
|
|
|
+ if not index_file.exists():
|
|
|
|
|
+ return []
|
|
|
|
|
+ try:
|
|
|
|
|
+ entries = json.loads(index_file.read_text(encoding="utf-8"))
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ return []
|
|
|
|
|
+
|
|
|
|
|
+ # Tokenise query
|
|
|
|
|
+ import re
|
|
|
|
|
+ query_tokens = set(re.sub(r"\W+", " ", query.lower()).split())
|
|
|
|
|
+ if not query_tokens:
|
|
|
|
|
+ return entries[:top_k]
|
|
|
|
|
+
|
|
|
|
|
+ scored: List[tuple] = []
|
|
|
|
|
+ for entry in entries:
|
|
|
|
|
+ haystack = f"{entry.get('summary','')} {entry.get('text','')}".lower()
|
|
|
|
|
+ score = sum(1 for tok in query_tokens if tok in haystack) / max(len(query_tokens), 1)
|
|
|
|
|
+ scored.append((score, entry))
|
|
|
|
|
+
|
|
|
|
|
+ scored.sort(key=lambda x: -x[0])
|
|
|
|
|
+ results = []
|
|
|
|
|
+ for score, entry in scored[:top_k]:
|
|
|
|
|
+ results.append({**entry, "score": round(score, 4)})
|
|
|
|
|
+ return results
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
# ─── Search helpers ───
|
|
# ─── Search helpers ───
|
|
|
|
|
|
|
|
def _search_subprocess(kb_root: str, scripts_dir: Path, mode: str, query: str, top_k: int) -> List[Dict]:
|
|
def _search_subprocess(kb_root: str, scripts_dir: Path, mode: str, query: str, top_k: int) -> List[Dict]:
|
|
@@ -1411,11 +1565,15 @@ async def search_kb(
|
|
|
|
|
|
|
|
try:
|
|
try:
|
|
|
if req.mode == "fusion":
|
|
if req.mode == "fusion":
|
|
|
- # RAG Fusion: expand query → multi-search → RRF
|
|
|
|
|
results = await asyncio.get_event_loop().run_in_executor(
|
|
results = await asyncio.get_event_loop().run_in_executor(
|
|
|
None,
|
|
None,
|
|
|
lambda: _fusion_search(kb_root, scripts_dir, req.query, req.top_k),
|
|
lambda: _fusion_search(kb_root, scripts_dir, req.query, req.top_k),
|
|
|
)
|
|
)
|
|
|
|
|
+ elif req.mode == "pageindex":
|
|
|
|
|
+ results = await asyncio.get_event_loop().run_in_executor(
|
|
|
|
|
+ None,
|
|
|
|
|
+ lambda: _page_index_search(kb_root, req.query, req.top_k),
|
|
|
|
|
+ )
|
|
|
else:
|
|
else:
|
|
|
results = await asyncio.get_event_loop().run_in_executor(
|
|
results = await asyncio.get_event_loop().run_in_executor(
|
|
|
None,
|
|
None,
|
|
@@ -1613,7 +1771,18 @@ async def search_stream(
|
|
|
await asyncio.sleep(0)
|
|
await asyncio.sleep(0)
|
|
|
|
|
|
|
|
chunks: List[Dict] = []
|
|
chunks: List[Dict] = []
|
|
|
- if scripts_dir and (Path(kb_root) / "rag_index.json").exists():
|
|
|
|
|
|
|
+ index_exists = (Path(kb_root) / "rag_index.json").exists()
|
|
|
|
|
+ page_index_exists = (Path(kb_root) / "rag_page_index.json").exists()
|
|
|
|
|
+
|
|
|
|
|
+ if mode == "pageindex" and page_index_exists:
|
|
|
|
|
+ try:
|
|
|
|
|
+ chunks = await asyncio.get_event_loop().run_in_executor(
|
|
|
|
|
+ None,
|
|
|
|
|
+ lambda: _page_index_search(kb_root, q, top_k)
|
|
|
|
|
+ )
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ yield f"data: {json.dumps({'type': 'status', 'text': f'PageIndex 检索出错: {e}'})}\n\n"
|
|
|
|
|
+ elif scripts_dir and index_exists:
|
|
|
try:
|
|
try:
|
|
|
if mode == "fusion":
|
|
if mode == "fusion":
|
|
|
chunks = await asyncio.get_event_loop().run_in_executor(
|
|
chunks = await asyncio.get_event_loop().run_in_executor(
|
|
@@ -1628,7 +1797,8 @@ async def search_stream(
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
yield f"data: {json.dumps({'type': 'status', 'text': f'检索出错: {e}'})}\n\n"
|
|
yield f"data: {json.dumps({'type': 'status', 'text': f'检索出错: {e}'})}\n\n"
|
|
|
else:
|
|
else:
|
|
|
- yield f"data: {json.dumps({'type': 'status', 'text': 'BM25 索引不存在,请先构建索引'})}\n\n"
|
|
|
|
|
|
|
+ msg = "PageIndex 不存在,请先以 page 分块策略构建索引" if mode == "pageindex" else "BM25 索引不存在,请先构建索引"
|
|
|
|
|
+ yield f"data: {json.dumps({'type': 'status', 'text': msg})}\n\n"
|
|
|
|
|
|
|
|
# Emit retrieved chunks so the UI can show sources panel
|
|
# Emit retrieved chunks so the UI can show sources panel
|
|
|
yield f"data: {json.dumps({'type': 'chunks', 'chunks': chunks[:top_k]})}\n\n"
|
|
yield f"data: {json.dumps({'type': 'chunks', 'chunks': chunks[:top_k]})}\n\n"
|
|
@@ -1688,7 +1858,8 @@ async def search_stream(
|
|
|
try:
|
|
try:
|
|
|
loop = asyncio.get_event_loop()
|
|
loop = asyncio.get_event_loop()
|
|
|
# Use asyncio.Queue so the thread can push tokens without blocking the event loop
|
|
# Use asyncio.Queue so the thread can push tokens without blocking the event loop
|
|
|
- aio_q: asyncio.Queue = asyncio.Queue()
|
|
|
|
|
|
|
+ # FIX-09: bounded queue prevents OOM from slow clients
|
|
|
|
|
+ aio_q: asyncio.Queue = asyncio.Queue(maxsize=1000)
|
|
|
|
|
|
|
|
def _fill_q():
|
|
def _fill_q():
|
|
|
"""Runs in a thread — pushes (kind, val) into the asyncio queue."""
|
|
"""Runs in a thread — pushes (kind, val) into the asyncio queue."""
|