"""工具服务 HTTP 接口(FastAPI)。 这是“工具执行层”的唯一入口,BFF 与前端都通过它访问数据: =========================== ==================================================== 接口 说明 =========================== ==================================================== ``GET /health`` 健康检查 ``GET /meta`` 工具清单与版本(前端“关于”页使用) ``POST /workspaces`` 创建工作区 ``GET /workspaces`` 工作区列表 ``POST /workspaces/{id}/objects`` 上传原始文件(multipart) ``GET /objects/{id}`` 对象元数据(含表清单、行数、血缘) ``GET /objects/{id}/preview`` 预览某张表 ``GET /objects/{id}/file`` 下载对象内的文件(xlsx / csv) ``POST /runs`` 起一个处理任务(前端数据处理工作区) ``GET /runs/{id}`` 查询进度 ``POST /tools/data_read`` data_read 工具 ``POST /tools/calculator`` calculator 工具 ``POST /tools/customs_process`` 同步执行处理(Agent 工具) ``GET /ledger/...`` 台账查询与自检 =========================== ==================================================== """ from __future__ import annotations import io import json import zipfile from contextlib import asynccontextmanager from pathlib import Path from urllib.parse import quote from fastapi import Body, FastAPI, File, HTTPException, Query, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, JSONResponse, Response from pydantic import BaseModel, Field from . import __version__ from .calc import CalcError, calculate from .config import settings from .customs import OPERATIONS, load_processor from .db import database from .ledger import ledger from .objects import store from .readers import QueryTimeout, ReadError, read_rows from .runs import build_options, runs @asynccontextmanager async def lifespan(_app: FastAPI): """启动时建目录、建表、保证有一个默认工作区。""" settings.ensure() database # 触发建表 store.ensure_default_workspace() yield app = FastAPI(title="海关问数 Agent · 工具执行服务", version=__version__, lifespan=lifespan) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=False, allow_methods=["*"], allow_headers=["*"], ) # ---------------------------------------------------------------------- 基础 @app.get("/health") def health() -> dict: return {"ok": True, "service": "tools", "version": __version__, "data_dir": str(settings.data_dir)} @app.get("/meta") def meta() -> dict: processor = load_processor() with database.connect() as connection: workspaces = connection.execute("SELECT COUNT(*) FROM workspaces").fetchone()[0] objects = connection.execute("SELECT COUNT(*) FROM objects").fetchone()[0] return { "version": __version__, "processor": {"name": getattr(processor, "name", "custom"), "version": getattr(processor, "version", "0")}, "operations": list(OPERATIONS), "tools": ["customs_process", "data_read", "calculator"], "workspaces": workspaces, "objects": objects, "ledger": ledger.stats(), } # ---------------------------------------------------------------------- 工作区 class WorkspaceCreate(BaseModel): name: str = Field(min_length=1, max_length=80) description: str = "" @app.get("/workspaces") def list_workspaces() -> dict: return {"items": store.list_workspaces()} @app.post("/workspaces", status_code=201) def create_workspace(payload: WorkspaceCreate) -> dict: return store.create_workspace(payload.name, payload.description) @app.delete("/workspaces/{workspace_id}") def delete_workspace(workspace_id: str) -> dict: try: store.delete_workspace(workspace_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error return {"ok": True} # ---------------------------------------------------------------------- 对象 @app.get("/workspaces/{workspace_id}/objects") def list_objects(workspace_id: str, kind: str | None = None) -> dict: return {"items": store.list_objects(workspace_id, kind)} @app.post("/workspaces/{workspace_id}/objects", status_code=201) async def upload_object(workspace_id: str, file: UploadFile = File(...)) -> dict: payload = await file.read() if len(payload) > settings.max_upload_bytes: raise HTTPException(status_code=413, detail=f"文件超过上限 {settings.max_upload_bytes // 1024 // 1024} MB") try: return store.register_upload(workspace_id, file.filename or "upload.csv", payload) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error except Exception as error: # noqa: BLE001 raise HTTPException(status_code=400, detail=f"文件解析失败: {error}") from error @app.get("/objects/{object_id}") def get_object(object_id: str) -> dict: try: return store.get_object(object_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error @app.delete("/objects/{object_id}") def delete_object(object_id: str) -> dict: try: store.delete_object(object_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error return {"ok": True} @app.get("/objects/{object_id}/preview") def preview_object( object_id: str, table: str = "raw", limit: int = Query(20, ge=1, le=200), offset: int = Query(0, ge=0), ) -> dict: try: return store.preview(object_id, table, limit, offset) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error except QueryTimeout as error: raise HTTPException(status_code=408, detail=str(error)) from error except ReadError as error: raise HTTPException(status_code=400, detail=str(error)) from error @app.get("/objects/{object_id}/row") def object_row(object_id: str, table: str = "raw", row: int = Query(..., ge=0)) -> dict: """按行号取一行(溯源 chip 点击后要看到"这一行原来是什么")。 返回该行的原文、行哈希,以及行级血缘(``source_rows`` 谓词,可据此下钻到原始记录)。 """ try: return store.read_row(object_id, table, row) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error except QueryTimeout as error: raise HTTPException(status_code=408, detail=str(error)) from error except ReadError as error: raise HTTPException(status_code=400, detail=str(error)) from error @app.get("/objects/{object_id}/sheet_meta") def object_sheet_meta(object_id: str) -> dict: """结果表的自描述:每张表的行数、列名、行哈希数量(供前端"数据浏览"用)。""" try: item = store.get_object(object_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error return { "object_id": object_id, "name": item["name"], "kind": item["kind"], "rows": item["rows"], "tables": item["tables"], "lineage": item["meta"].get("lineage", {}), "parents": item["parents"], } @app.get("/objects/{object_id}/lineage") def object_lineage(object_id: str, table: str | None = None, row: int | None = None) -> dict: try: item = store.get_object(object_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error if table is not None and row is not None: entry = store.row_lineage(object_id, table, row) if not entry: raise HTTPException(status_code=404, detail="该行没有血缘记录") return entry return {"object_id": object_id, "parents": item["parents"], "lineage": item["meta"].get("lineage", {})} @app.get("/objects/{object_id}/file") def download_file(object_id: str, name: str = Query(..., description="对象目录内的相对文件名")): try: item = store.get_object(object_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error base = Path(item["path"]).parent if Path(item["path"]).is_file() else Path(item["path"]) target = (base / name).resolve() try: target.relative_to(base.resolve()) except ValueError as error: raise HTTPException(status_code=400, detail="非法路径") from error if not target.exists(): raise HTTPException(status_code=404, detail=f"文件不存在: {name}") return FileResponse(target, filename=target.name) @app.get("/objects/{object_id}/archive") def download_archive(object_id: str, include: str = Query("all", description="all / xlsx / csv")): """把一次处理的结果打包成 zip(Excel + 逐表 CSV + 明细),供"一次拿走全部结果文件"。""" try: item = store.get_object(object_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error base = Path(item["path"]) if not base.is_dir(): raise HTTPException(status_code=400, detail="该对象不是处理结果目录,无法打包") kinds = {"all": None, "xlsx": {".xlsx"}, "csv": {".csv", ".json"}}.get(include.lower(), None) buffer = io.BytesIO() added = 0 with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive: for path in sorted(base.rglob("*")): if not path.is_file(): continue if kinds is not None and path.suffix.lower() not in kinds: continue if path.name in {"working.sqlite", "working.sqlite-wal", "working.sqlite-shm"}: continue archive.write(path, arcname=str(path.relative_to(base)).replace("\\", "/")) added += 1 if added == 0: raise HTTPException(status_code=404, detail="没有可打包的文件") buffer.seek(0) filename = f"{(item.get('name') or object_id).rsplit('.', 1)[0]}_结果文件.zip" return Response( content=buffer.getvalue(), media_type="application/zip", headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"}, ) # ---------------------------------------------------------------------- 处理任务 class RunCreate(BaseModel): workspace_id: str input_object_id: str operation: str = "full_pipeline" options: dict = Field(default_factory=dict) output_name: str | None = None @app.post("/runs", status_code=201) def create_run(payload: RunCreate) -> dict: if payload.operation not in OPERATIONS: raise HTTPException(status_code=400, detail=f"operation 只支持 {OPERATIONS}") try: build_options(payload.options).validate() except ValueError as error: raise HTTPException(status_code=400, detail=str(error)) from error try: return runs.create(payload.workspace_id, payload.input_object_id, payload.operation, payload.options, payload.output_name) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error except ValueError as error: raise HTTPException(status_code=400, detail=str(error)) from error @app.get("/runs") def list_runs(limit: int = Query(20, ge=1, le=100)) -> dict: return {"items": runs.list(limit)} @app.get("/runs/{run_id}") def get_run(run_id: str) -> dict: try: return runs.get(run_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error # ---------------------------------------------------------------------- 工具 class DataReadRequest(BaseModel): object_id: str table: str = "raw" columns: list[str] | None = None filters: list[dict] | None = None order_by: list[dict] | None = None limit: int | None = Field(default=None, gt=0) offset: int = 0 aggregate: dict | None = None count_mode: str = "auto" cursor: str | None = None use_cache: bool = True #: 由 BFF 传入:把本次调用登记进数据面台账,call_id 与溯源令牌一致 call_id: str = "" session_id: str = "" @app.post("/tools/data_read") def tool_data_read(payload: DataReadRequest) -> dict: try: result = read_rows( payload.object_id, table=payload.table, filters=payload.filters, columns=payload.columns, order_by=payload.order_by, limit=payload.limit, offset=payload.offset, aggregate=payload.aggregate, count_mode=payload.count_mode, cursor=payload.cursor, use_cache=payload.use_cache, ) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error except QueryTimeout as error: raise HTTPException(status_code=408, detail=str(error)) from error except ReadError as error: raise HTTPException(status_code=400, detail=str(error)) from error if payload.call_id: # 数据面台账:调用级记录 + 行级索引,供离线审计和"这个数从哪来" ledger.record( payload.call_id, "data_read", { "object_id": payload.object_id, "table": payload.table, "filters": payload.filters, "limit": payload.limit, "offset": payload.offset, "aggregate": payload.aggregate, "columns": payload.columns, "order_by": payload.order_by, "count_mode": payload.count_mode, "cursor": payload.cursor, "query_version": result.get("query_version"), }, { "columns": result["columns"], "total": result.get("total"), "returned": len(result["rows"]), }, object_ids=[payload.object_id], session_id=payload.session_id, rows=[ { "values": row["values"], "_trace": { "object_id": row["_trace"].get("object_id", payload.object_id), "table": row["_trace"].get("table", payload.table), "row_index": row["_trace"].get("row_index", -1), "row_hash": row["_trace"].get("row_hash", ""), }, } for row in result["rows"] ], ) return result class CalcRequest(BaseModel): op: str = "sum" inputs: list[dict] | None = None expression: str | None = None session_id: str = "" label: str = "" @app.post("/tools/calculator") def tool_calculator(payload: CalcRequest) -> dict: try: return calculate(payload.op, payload.inputs, payload.expression, session_id=payload.session_id, label=payload.label) except CalcError as error: raise HTTPException(status_code=400, detail=str(error)) from error class ProcessRequest(BaseModel): workspace_id: str input_object_id: str operation: str = "full_pipeline" params: dict = Field(default_factory=dict) output_filename: str | None = None call_id: str = "" session_id: str = "" @app.post("/tools/customs_process") def tool_customs_process(payload: ProcessRequest) -> dict: if payload.operation not in OPERATIONS: raise HTTPException(status_code=400, detail=f"operation 只支持 {OPERATIONS}") try: run = runs.run_sync(payload.workspace_id, payload.input_object_id, payload.operation, payload.params, payload.output_filename) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error if run["status"] != "succeeded": return JSONResponse( status_code=422, content={"detail": run.get("error") or "处理失败", "run": run}, ) item = store.get_object(run["object_id"]) if payload.call_id: ledger.record( payload.call_id, "customs_process", { "workspace_id": payload.workspace_id, "input_object_id": payload.input_object_id, "operation": payload.operation, "params": payload.params, }, { "run_id": run["id"], "object_id": item["id"], "sheet_count": len(item["tables"]), "rows": item["rows"], "lineage": item["meta"].get("lineage", {}), }, object_ids=[payload.input_object_id, item["id"]], session_id=payload.session_id, ) return { "run_id": run["id"], "object_id": item["id"], "name": item["name"], "sheets": item["tables"], "lineage": item["meta"].get("lineage", {}), "stats": item["meta"].get("stats", {}), "warnings": item["meta"].get("warnings", []), "files": _object_files(item), } def _object_files(item: dict) -> list[str]: base = Path(item["path"]) directory = base.parent if base.is_file() else base files = [] for path in sorted(directory.rglob("*")): if path.is_file() and path.suffix.lower() in {".xlsx", ".csv", ".json", ".sqlite"}: files.append(str(path.relative_to(directory)).replace("\\", "/")) return files[:200] # ---------------------------------------------------------------------- 台账 @app.get("/ledger/calls") def ledger_calls(session_id: str = "") -> dict: return {"items": ledger.session_calls(session_id)} @app.get("/ledger/value") def ledger_value(value: str, session_id: str = "") -> dict: return {"items": ledger.find_value(value, session_id)} @app.get("/ledger/stats") def ledger_stats() -> dict: return ledger.stats() @app.get("/ledger/call/{call_id}") def ledger_call(call_id: str) -> dict: item = ledger.get(call_id) if not item: raise HTTPException(status_code=404, detail="调用不存在") return item @app.get("/ledger/export") def ledger_export() -> dict: """导出 JSONL 台账(离线审计)。""" if not settings.ledger_path.exists(): return {"items": []} lines = settings.ledger_path.read_text(encoding="utf-8").strip().splitlines() return {"items": [json.loads(line) for line in lines if line.strip()]} @app.exception_handler(Exception) def unhandled_error(_request, error: Exception): # noqa: ANN001 return JSONResponse(status_code=500, content={"detail": f"{type(error).__name__}: {error}"})