api.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512
  1. """工具服务 HTTP 接口(FastAPI)。
  2. 这是“工具执行层”的唯一入口,BFF 与前端都通过它访问数据:
  3. =========================== ====================================================
  4. 接口 说明
  5. =========================== ====================================================
  6. ``GET /health`` 健康检查
  7. ``GET /meta`` 工具清单与版本(前端“关于”页使用)
  8. ``POST /workspaces`` 创建工作区
  9. ``GET /workspaces`` 工作区列表
  10. ``POST /workspaces/{id}/objects`` 上传原始文件(multipart)
  11. ``GET /objects/{id}`` 对象元数据(含表清单、行数、血缘)
  12. ``GET /objects/{id}/preview`` 预览某张表
  13. ``GET /objects/{id}/file`` 下载对象内的文件(xlsx / csv)
  14. ``POST /runs`` 起一个处理任务(前端数据处理工作区)
  15. ``GET /runs/{id}`` 查询进度
  16. ``POST /tools/data_read`` data_read 工具
  17. ``POST /tools/calculator`` calculator 工具
  18. ``POST /tools/customs_process`` 同步执行处理(Agent 工具)
  19. ``GET /ledger/...`` 台账查询与自检
  20. =========================== ====================================================
  21. """
  22. from __future__ import annotations
  23. import io
  24. import json
  25. import zipfile
  26. from contextlib import asynccontextmanager
  27. from pathlib import Path
  28. from urllib.parse import quote
  29. from fastapi import Body, FastAPI, File, HTTPException, Query, UploadFile
  30. from fastapi.middleware.cors import CORSMiddleware
  31. from fastapi.responses import FileResponse, JSONResponse, Response
  32. from pydantic import BaseModel, Field
  33. from . import __version__
  34. from .calc import CalcError, calculate
  35. from .config import settings
  36. from .customs import OPERATIONS, load_processor
  37. from .db import database
  38. from .ledger import ledger
  39. from .objects import store
  40. from .readers import QueryTimeout, ReadError, read_rows
  41. from .runs import build_options, runs
  42. @asynccontextmanager
  43. async def lifespan(_app: FastAPI):
  44. """启动时建目录、建表、保证有一个默认工作区。"""
  45. settings.ensure()
  46. database # 触发建表
  47. store.ensure_default_workspace()
  48. yield
  49. app = FastAPI(title="海关问数 Agent · 工具执行服务", version=__version__, lifespan=lifespan)
  50. app.add_middleware(
  51. CORSMiddleware,
  52. allow_origins=["*"],
  53. allow_credentials=False,
  54. allow_methods=["*"],
  55. allow_headers=["*"],
  56. )
  57. # ---------------------------------------------------------------------- 基础
  58. @app.get("/health")
  59. def health() -> dict:
  60. return {"ok": True, "service": "tools", "version": __version__, "data_dir": str(settings.data_dir)}
  61. @app.get("/meta")
  62. def meta() -> dict:
  63. processor = load_processor()
  64. with database.connect() as connection:
  65. workspaces = connection.execute("SELECT COUNT(*) FROM workspaces").fetchone()[0]
  66. objects = connection.execute("SELECT COUNT(*) FROM objects").fetchone()[0]
  67. return {
  68. "version": __version__,
  69. "processor": {"name": getattr(processor, "name", "custom"), "version": getattr(processor, "version", "0")},
  70. "operations": list(OPERATIONS),
  71. "tools": ["customs_process", "data_read", "calculator"],
  72. "workspaces": workspaces,
  73. "objects": objects,
  74. "ledger": ledger.stats(),
  75. }
  76. # ---------------------------------------------------------------------- 工作区
  77. class WorkspaceCreate(BaseModel):
  78. name: str = Field(min_length=1, max_length=80)
  79. description: str = ""
  80. @app.get("/workspaces")
  81. def list_workspaces() -> dict:
  82. return {"items": store.list_workspaces()}
  83. @app.post("/workspaces", status_code=201)
  84. def create_workspace(payload: WorkspaceCreate) -> dict:
  85. return store.create_workspace(payload.name, payload.description)
  86. @app.delete("/workspaces/{workspace_id}")
  87. def delete_workspace(workspace_id: str) -> dict:
  88. try:
  89. store.delete_workspace(workspace_id)
  90. except KeyError as error:
  91. raise HTTPException(status_code=404, detail=str(error)) from error
  92. return {"ok": True}
  93. # ---------------------------------------------------------------------- 对象
  94. @app.get("/workspaces/{workspace_id}/objects")
  95. def list_objects(workspace_id: str, kind: str | None = None) -> dict:
  96. return {"items": store.list_objects(workspace_id, kind)}
  97. @app.post("/workspaces/{workspace_id}/objects", status_code=201)
  98. async def upload_object(workspace_id: str, file: UploadFile = File(...)) -> dict:
  99. payload = await file.read()
  100. if len(payload) > settings.max_upload_bytes:
  101. raise HTTPException(status_code=413, detail=f"文件超过上限 {settings.max_upload_bytes // 1024 // 1024} MB")
  102. try:
  103. return store.register_upload(workspace_id, file.filename or "upload.csv", payload)
  104. except KeyError as error:
  105. raise HTTPException(status_code=404, detail=str(error)) from error
  106. except Exception as error: # noqa: BLE001
  107. raise HTTPException(status_code=400, detail=f"文件解析失败: {error}") from error
  108. @app.get("/objects/{object_id}")
  109. def get_object(object_id: str) -> dict:
  110. try:
  111. return store.get_object(object_id)
  112. except KeyError as error:
  113. raise HTTPException(status_code=404, detail=str(error)) from error
  114. @app.delete("/objects/{object_id}")
  115. def delete_object(object_id: str) -> dict:
  116. try:
  117. store.delete_object(object_id)
  118. except KeyError as error:
  119. raise HTTPException(status_code=404, detail=str(error)) from error
  120. return {"ok": True}
  121. @app.get("/objects/{object_id}/preview")
  122. def preview_object(
  123. object_id: str,
  124. table: str = "raw",
  125. limit: int = Query(20, ge=1, le=200),
  126. offset: int = Query(0, ge=0),
  127. ) -> dict:
  128. try:
  129. return store.preview(object_id, table, limit, offset)
  130. except KeyError as error:
  131. raise HTTPException(status_code=404, detail=str(error)) from error
  132. except QueryTimeout as error:
  133. raise HTTPException(status_code=408, detail=str(error)) from error
  134. except ReadError as error:
  135. raise HTTPException(status_code=400, detail=str(error)) from error
  136. @app.get("/objects/{object_id}/row")
  137. def object_row(object_id: str, table: str = "raw", row: int = Query(..., ge=0)) -> dict:
  138. """按行号取一行(溯源 chip 点击后要看到"这一行原来是什么")。
  139. 返回该行的原文、行哈希,以及行级血缘(``source_rows`` 谓词,可据此下钻到原始记录)。
  140. """
  141. try:
  142. return store.read_row(object_id, table, row)
  143. except KeyError as error:
  144. raise HTTPException(status_code=404, detail=str(error)) from error
  145. except QueryTimeout as error:
  146. raise HTTPException(status_code=408, detail=str(error)) from error
  147. except ReadError as error:
  148. raise HTTPException(status_code=400, detail=str(error)) from error
  149. @app.get("/objects/{object_id}/sheet_meta")
  150. def object_sheet_meta(object_id: str) -> dict:
  151. """结果表的自描述:每张表的行数、列名、行哈希数量(供前端"数据浏览"用)。"""
  152. try:
  153. item = store.get_object(object_id)
  154. except KeyError as error:
  155. raise HTTPException(status_code=404, detail=str(error)) from error
  156. return {
  157. "object_id": object_id,
  158. "name": item["name"],
  159. "kind": item["kind"],
  160. "rows": item["rows"],
  161. "tables": item["tables"],
  162. "lineage": item["meta"].get("lineage", {}),
  163. "parents": item["parents"],
  164. }
  165. @app.get("/objects/{object_id}/lineage")
  166. def object_lineage(object_id: str, table: str | None = None, row: int | None = None) -> dict:
  167. try:
  168. item = store.get_object(object_id)
  169. except KeyError as error:
  170. raise HTTPException(status_code=404, detail=str(error)) from error
  171. if table is not None and row is not None:
  172. entry = store.row_lineage(object_id, table, row)
  173. if not entry:
  174. raise HTTPException(status_code=404, detail="该行没有血缘记录")
  175. return entry
  176. return {"object_id": object_id, "parents": item["parents"], "lineage": item["meta"].get("lineage", {})}
  177. @app.get("/objects/{object_id}/file")
  178. def download_file(object_id: str, name: str = Query(..., description="对象目录内的相对文件名")):
  179. try:
  180. item = store.get_object(object_id)
  181. except KeyError as error:
  182. raise HTTPException(status_code=404, detail=str(error)) from error
  183. base = Path(item["path"]).parent if Path(item["path"]).is_file() else Path(item["path"])
  184. target = (base / name).resolve()
  185. try:
  186. target.relative_to(base.resolve())
  187. except ValueError as error:
  188. raise HTTPException(status_code=400, detail="非法路径") from error
  189. if not target.exists():
  190. raise HTTPException(status_code=404, detail=f"文件不存在: {name}")
  191. return FileResponse(target, filename=target.name)
  192. @app.get("/objects/{object_id}/archive")
  193. def download_archive(object_id: str, include: str = Query("all", description="all / xlsx / csv")):
  194. """把一次处理的结果打包成 zip(Excel + 逐表 CSV + 明细),供"一次拿走全部结果文件"。"""
  195. try:
  196. item = store.get_object(object_id)
  197. except KeyError as error:
  198. raise HTTPException(status_code=404, detail=str(error)) from error
  199. base = Path(item["path"])
  200. if not base.is_dir():
  201. raise HTTPException(status_code=400, detail="该对象不是处理结果目录,无法打包")
  202. kinds = {"all": None, "xlsx": {".xlsx"}, "csv": {".csv", ".json"}}.get(include.lower(), None)
  203. buffer = io.BytesIO()
  204. added = 0
  205. with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
  206. for path in sorted(base.rglob("*")):
  207. if not path.is_file():
  208. continue
  209. if kinds is not None and path.suffix.lower() not in kinds:
  210. continue
  211. if path.name in {"working.sqlite", "working.sqlite-wal", "working.sqlite-shm"}:
  212. continue
  213. archive.write(path, arcname=str(path.relative_to(base)).replace("\\", "/"))
  214. added += 1
  215. if added == 0:
  216. raise HTTPException(status_code=404, detail="没有可打包的文件")
  217. buffer.seek(0)
  218. filename = f"{(item.get('name') or object_id).rsplit('.', 1)[0]}_结果文件.zip"
  219. return Response(
  220. content=buffer.getvalue(),
  221. media_type="application/zip",
  222. headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
  223. )
  224. # ---------------------------------------------------------------------- 处理任务
  225. class RunCreate(BaseModel):
  226. workspace_id: str
  227. input_object_id: str
  228. operation: str = "full_pipeline"
  229. options: dict = Field(default_factory=dict)
  230. output_name: str | None = None
  231. @app.post("/runs", status_code=201)
  232. def create_run(payload: RunCreate) -> dict:
  233. if payload.operation not in OPERATIONS:
  234. raise HTTPException(status_code=400, detail=f"operation 只支持 {OPERATIONS}")
  235. try:
  236. build_options(payload.options).validate()
  237. except ValueError as error:
  238. raise HTTPException(status_code=400, detail=str(error)) from error
  239. try:
  240. return runs.create(payload.workspace_id, payload.input_object_id, payload.operation, payload.options, payload.output_name)
  241. except KeyError as error:
  242. raise HTTPException(status_code=404, detail=str(error)) from error
  243. except ValueError as error:
  244. raise HTTPException(status_code=400, detail=str(error)) from error
  245. @app.get("/runs")
  246. def list_runs(limit: int = Query(20, ge=1, le=100)) -> dict:
  247. return {"items": runs.list(limit)}
  248. @app.get("/runs/{run_id}")
  249. def get_run(run_id: str) -> dict:
  250. try:
  251. return runs.get(run_id)
  252. except KeyError as error:
  253. raise HTTPException(status_code=404, detail=str(error)) from error
  254. # ---------------------------------------------------------------------- 工具
  255. class DataReadRequest(BaseModel):
  256. object_id: str
  257. table: str = "raw"
  258. columns: list[str] | None = None
  259. filters: list[dict] | None = None
  260. order_by: list[dict] | None = None
  261. limit: int | None = Field(default=None, gt=0)
  262. offset: int = 0
  263. aggregate: dict | None = None
  264. count_mode: str = "auto"
  265. cursor: str | None = None
  266. use_cache: bool = True
  267. #: 由 BFF 传入:把本次调用登记进数据面台账,call_id 与溯源令牌一致
  268. call_id: str = ""
  269. session_id: str = ""
  270. @app.post("/tools/data_read")
  271. def tool_data_read(payload: DataReadRequest) -> dict:
  272. try:
  273. result = read_rows(
  274. payload.object_id,
  275. table=payload.table,
  276. filters=payload.filters,
  277. columns=payload.columns,
  278. order_by=payload.order_by,
  279. limit=payload.limit,
  280. offset=payload.offset,
  281. aggregate=payload.aggregate,
  282. count_mode=payload.count_mode,
  283. cursor=payload.cursor,
  284. use_cache=payload.use_cache,
  285. )
  286. except KeyError as error:
  287. raise HTTPException(status_code=404, detail=str(error)) from error
  288. except QueryTimeout as error:
  289. raise HTTPException(status_code=408, detail=str(error)) from error
  290. except ReadError as error:
  291. raise HTTPException(status_code=400, detail=str(error)) from error
  292. if payload.call_id:
  293. # 数据面台账:调用级记录 + 行级索引,供离线审计和"这个数从哪来"
  294. ledger.record(
  295. payload.call_id,
  296. "data_read",
  297. {
  298. "object_id": payload.object_id,
  299. "table": payload.table,
  300. "filters": payload.filters,
  301. "limit": payload.limit,
  302. "offset": payload.offset,
  303. "aggregate": payload.aggregate,
  304. "columns": payload.columns,
  305. "order_by": payload.order_by,
  306. "count_mode": payload.count_mode,
  307. "cursor": payload.cursor,
  308. "query_version": result.get("query_version"),
  309. },
  310. {
  311. "columns": result["columns"],
  312. "total": result.get("total"),
  313. "returned": len(result["rows"]),
  314. },
  315. object_ids=[payload.object_id],
  316. session_id=payload.session_id,
  317. rows=[
  318. {
  319. "values": row["values"],
  320. "_trace": {
  321. "object_id": row["_trace"].get("object_id", payload.object_id),
  322. "table": row["_trace"].get("table", payload.table),
  323. "row_index": row["_trace"].get("row_index", -1),
  324. "row_hash": row["_trace"].get("row_hash", ""),
  325. },
  326. }
  327. for row in result["rows"]
  328. ],
  329. )
  330. return result
  331. class CalcRequest(BaseModel):
  332. op: str = "sum"
  333. inputs: list[dict] | None = None
  334. expression: str | None = None
  335. session_id: str = ""
  336. label: str = ""
  337. @app.post("/tools/calculator")
  338. def tool_calculator(payload: CalcRequest) -> dict:
  339. try:
  340. return calculate(payload.op, payload.inputs, payload.expression, session_id=payload.session_id, label=payload.label)
  341. except CalcError as error:
  342. raise HTTPException(status_code=400, detail=str(error)) from error
  343. class ProcessRequest(BaseModel):
  344. workspace_id: str
  345. input_object_id: str
  346. operation: str = "full_pipeline"
  347. params: dict = Field(default_factory=dict)
  348. output_filename: str | None = None
  349. call_id: str = ""
  350. session_id: str = ""
  351. @app.post("/tools/customs_process")
  352. def tool_customs_process(payload: ProcessRequest) -> dict:
  353. if payload.operation not in OPERATIONS:
  354. raise HTTPException(status_code=400, detail=f"operation 只支持 {OPERATIONS}")
  355. try:
  356. run = runs.run_sync(payload.workspace_id, payload.input_object_id, payload.operation, payload.params, payload.output_filename)
  357. except KeyError as error:
  358. raise HTTPException(status_code=404, detail=str(error)) from error
  359. if run["status"] != "succeeded":
  360. return JSONResponse(
  361. status_code=422,
  362. content={"detail": run.get("error") or "处理失败", "run": run},
  363. )
  364. item = store.get_object(run["object_id"])
  365. if payload.call_id:
  366. ledger.record(
  367. payload.call_id,
  368. "customs_process",
  369. {
  370. "workspace_id": payload.workspace_id,
  371. "input_object_id": payload.input_object_id,
  372. "operation": payload.operation,
  373. "params": payload.params,
  374. },
  375. {
  376. "run_id": run["id"],
  377. "object_id": item["id"],
  378. "sheet_count": len(item["tables"]),
  379. "rows": item["rows"],
  380. "lineage": item["meta"].get("lineage", {}),
  381. },
  382. object_ids=[payload.input_object_id, item["id"]],
  383. session_id=payload.session_id,
  384. )
  385. return {
  386. "run_id": run["id"],
  387. "object_id": item["id"],
  388. "name": item["name"],
  389. "sheets": item["tables"],
  390. "lineage": item["meta"].get("lineage", {}),
  391. "stats": item["meta"].get("stats", {}),
  392. "warnings": item["meta"].get("warnings", []),
  393. "files": _object_files(item),
  394. }
  395. def _object_files(item: dict) -> list[str]:
  396. base = Path(item["path"])
  397. directory = base.parent if base.is_file() else base
  398. files = []
  399. for path in sorted(directory.rglob("*")):
  400. if path.is_file() and path.suffix.lower() in {".xlsx", ".csv", ".json", ".sqlite"}:
  401. files.append(str(path.relative_to(directory)).replace("\\", "/"))
  402. return files[:200]
  403. # ---------------------------------------------------------------------- 台账
  404. @app.get("/ledger/calls")
  405. def ledger_calls(session_id: str = "") -> dict:
  406. return {"items": ledger.session_calls(session_id)}
  407. @app.get("/ledger/value")
  408. def ledger_value(value: str, session_id: str = "") -> dict:
  409. return {"items": ledger.find_value(value, session_id)}
  410. @app.get("/ledger/stats")
  411. def ledger_stats() -> dict:
  412. return ledger.stats()
  413. @app.get("/ledger/call/{call_id}")
  414. def ledger_call(call_id: str) -> dict:
  415. item = ledger.get(call_id)
  416. if not item:
  417. raise HTTPException(status_code=404, detail="调用不存在")
  418. return item
  419. @app.get("/ledger/export")
  420. def ledger_export() -> dict:
  421. """导出 JSONL 台账(离线审计)。"""
  422. if not settings.ledger_path.exists():
  423. return {"items": []}
  424. lines = settings.ledger_path.read_text(encoding="utf-8").strip().splitlines()
  425. return {"items": [json.loads(line) for line in lines if line.strip()]}
  426. @app.exception_handler(Exception)
  427. def unhandled_error(_request, error: Exception): # noqa: ANN001
  428. return JSONResponse(status_code=500, content={"detail": f"{type(error).__name__}: {error}"})