| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512 |
- """工具服务 HTTP 接口(FastAPI)。
- 这是“工具执行层”的唯一入口,BFF 与前端都通过它访问数据:
- =========================== ====================================================
- 接口 说明
- =========================== ====================================================
- ``GET /health`` 健康检查
- ``GET /meta`` 工具清单与版本(前端“关于”页使用)
- ``POST /workspaces`` 创建工作区
- ``GET /workspaces`` 工作区列表
- ``POST /workspaces/{id}/objects`` 上传原始文件(multipart)
- ``GET /objects/{id}`` 对象元数据(含表清单、行数、血缘)
- ``GET /objects/{id}/preview`` 预览某张表
- ``GET /objects/{id}/file`` 下载对象内的文件(xlsx / csv)
- ``POST /runs`` 起一个处理任务(前端数据处理工作区)
- ``GET /runs/{id}`` 查询进度
- ``POST /tools/data_read`` data_read 工具
- ``POST /tools/calculator`` calculator 工具
- ``POST /tools/customs_process`` 同步执行处理(Agent 工具)
- ``GET /ledger/...`` 台账查询与自检
- =========================== ====================================================
- """
- from __future__ import annotations
- import io
- import json
- import zipfile
- from contextlib import asynccontextmanager
- from pathlib import Path
- from urllib.parse import quote
- from fastapi import Body, FastAPI, File, HTTPException, Query, UploadFile
- from fastapi.middleware.cors import CORSMiddleware
- from fastapi.responses import FileResponse, JSONResponse, Response
- from pydantic import BaseModel, Field
- from . import __version__
- from .calc import CalcError, calculate
- from .config import settings
- from .customs import OPERATIONS, load_processor
- from .db import database
- from .ledger import ledger
- from .objects import store
- from .readers import QueryTimeout, ReadError, read_rows
- from .runs import build_options, runs
- @asynccontextmanager
- async def lifespan(_app: FastAPI):
- """启动时建目录、建表、保证有一个默认工作区。"""
- settings.ensure()
- database # 触发建表
- store.ensure_default_workspace()
- yield
- app = FastAPI(title="海关问数 Agent · 工具执行服务", version=__version__, lifespan=lifespan)
- app.add_middleware(
- CORSMiddleware,
- allow_origins=["*"],
- allow_credentials=False,
- allow_methods=["*"],
- allow_headers=["*"],
- )
- # ---------------------------------------------------------------------- 基础
- @app.get("/health")
- def health() -> dict:
- return {"ok": True, "service": "tools", "version": __version__, "data_dir": str(settings.data_dir)}
- @app.get("/meta")
- def meta() -> dict:
- processor = load_processor()
- with database.connect() as connection:
- workspaces = connection.execute("SELECT COUNT(*) FROM workspaces").fetchone()[0]
- objects = connection.execute("SELECT COUNT(*) FROM objects").fetchone()[0]
- return {
- "version": __version__,
- "processor": {"name": getattr(processor, "name", "custom"), "version": getattr(processor, "version", "0")},
- "operations": list(OPERATIONS),
- "tools": ["customs_process", "data_read", "calculator"],
- "workspaces": workspaces,
- "objects": objects,
- "ledger": ledger.stats(),
- }
- # ---------------------------------------------------------------------- 工作区
- class WorkspaceCreate(BaseModel):
- name: str = Field(min_length=1, max_length=80)
- description: str = ""
- @app.get("/workspaces")
- def list_workspaces() -> dict:
- return {"items": store.list_workspaces()}
- @app.post("/workspaces", status_code=201)
- def create_workspace(payload: WorkspaceCreate) -> dict:
- return store.create_workspace(payload.name, payload.description)
- @app.delete("/workspaces/{workspace_id}")
- def delete_workspace(workspace_id: str) -> dict:
- try:
- store.delete_workspace(workspace_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- return {"ok": True}
- # ---------------------------------------------------------------------- 对象
- @app.get("/workspaces/{workspace_id}/objects")
- def list_objects(workspace_id: str, kind: str | None = None) -> dict:
- return {"items": store.list_objects(workspace_id, kind)}
- @app.post("/workspaces/{workspace_id}/objects", status_code=201)
- async def upload_object(workspace_id: str, file: UploadFile = File(...)) -> dict:
- payload = await file.read()
- if len(payload) > settings.max_upload_bytes:
- raise HTTPException(status_code=413, detail=f"文件超过上限 {settings.max_upload_bytes // 1024 // 1024} MB")
- try:
- return store.register_upload(workspace_id, file.filename or "upload.csv", payload)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- except Exception as error: # noqa: BLE001
- raise HTTPException(status_code=400, detail=f"文件解析失败: {error}") from error
- @app.get("/objects/{object_id}")
- def get_object(object_id: str) -> dict:
- try:
- return store.get_object(object_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- @app.delete("/objects/{object_id}")
- def delete_object(object_id: str) -> dict:
- try:
- store.delete_object(object_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- return {"ok": True}
- @app.get("/objects/{object_id}/preview")
- def preview_object(
- object_id: str,
- table: str = "raw",
- limit: int = Query(20, ge=1, le=200),
- offset: int = Query(0, ge=0),
- ) -> dict:
- try:
- return store.preview(object_id, table, limit, offset)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- except QueryTimeout as error:
- raise HTTPException(status_code=408, detail=str(error)) from error
- except ReadError as error:
- raise HTTPException(status_code=400, detail=str(error)) from error
- @app.get("/objects/{object_id}/row")
- def object_row(object_id: str, table: str = "raw", row: int = Query(..., ge=0)) -> dict:
- """按行号取一行(溯源 chip 点击后要看到"这一行原来是什么")。
- 返回该行的原文、行哈希,以及行级血缘(``source_rows`` 谓词,可据此下钻到原始记录)。
- """
- try:
- return store.read_row(object_id, table, row)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- except QueryTimeout as error:
- raise HTTPException(status_code=408, detail=str(error)) from error
- except ReadError as error:
- raise HTTPException(status_code=400, detail=str(error)) from error
- @app.get("/objects/{object_id}/sheet_meta")
- def object_sheet_meta(object_id: str) -> dict:
- """结果表的自描述:每张表的行数、列名、行哈希数量(供前端"数据浏览"用)。"""
- try:
- item = store.get_object(object_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- return {
- "object_id": object_id,
- "name": item["name"],
- "kind": item["kind"],
- "rows": item["rows"],
- "tables": item["tables"],
- "lineage": item["meta"].get("lineage", {}),
- "parents": item["parents"],
- }
- @app.get("/objects/{object_id}/lineage")
- def object_lineage(object_id: str, table: str | None = None, row: int | None = None) -> dict:
- try:
- item = store.get_object(object_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- if table is not None and row is not None:
- entry = store.row_lineage(object_id, table, row)
- if not entry:
- raise HTTPException(status_code=404, detail="该行没有血缘记录")
- return entry
- return {"object_id": object_id, "parents": item["parents"], "lineage": item["meta"].get("lineage", {})}
- @app.get("/objects/{object_id}/file")
- def download_file(object_id: str, name: str = Query(..., description="对象目录内的相对文件名")):
- try:
- item = store.get_object(object_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- base = Path(item["path"]).parent if Path(item["path"]).is_file() else Path(item["path"])
- target = (base / name).resolve()
- try:
- target.relative_to(base.resolve())
- except ValueError as error:
- raise HTTPException(status_code=400, detail="非法路径") from error
- if not target.exists():
- raise HTTPException(status_code=404, detail=f"文件不存在: {name}")
- return FileResponse(target, filename=target.name)
- @app.get("/objects/{object_id}/archive")
- def download_archive(object_id: str, include: str = Query("all", description="all / xlsx / csv")):
- """把一次处理的结果打包成 zip(Excel + 逐表 CSV + 明细),供"一次拿走全部结果文件"。"""
- try:
- item = store.get_object(object_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- base = Path(item["path"])
- if not base.is_dir():
- raise HTTPException(status_code=400, detail="该对象不是处理结果目录,无法打包")
- kinds = {"all": None, "xlsx": {".xlsx"}, "csv": {".csv", ".json"}}.get(include.lower(), None)
- buffer = io.BytesIO()
- added = 0
- with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
- for path in sorted(base.rglob("*")):
- if not path.is_file():
- continue
- if kinds is not None and path.suffix.lower() not in kinds:
- continue
- if path.name in {"working.sqlite", "working.sqlite-wal", "working.sqlite-shm"}:
- continue
- archive.write(path, arcname=str(path.relative_to(base)).replace("\\", "/"))
- added += 1
- if added == 0:
- raise HTTPException(status_code=404, detail="没有可打包的文件")
- buffer.seek(0)
- filename = f"{(item.get('name') or object_id).rsplit('.', 1)[0]}_结果文件.zip"
- return Response(
- content=buffer.getvalue(),
- media_type="application/zip",
- headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
- )
- # ---------------------------------------------------------------------- 处理任务
- class RunCreate(BaseModel):
- workspace_id: str
- input_object_id: str
- operation: str = "full_pipeline"
- options: dict = Field(default_factory=dict)
- output_name: str | None = None
- @app.post("/runs", status_code=201)
- def create_run(payload: RunCreate) -> dict:
- if payload.operation not in OPERATIONS:
- raise HTTPException(status_code=400, detail=f"operation 只支持 {OPERATIONS}")
- try:
- build_options(payload.options).validate()
- except ValueError as error:
- raise HTTPException(status_code=400, detail=str(error)) from error
- try:
- return runs.create(payload.workspace_id, payload.input_object_id, payload.operation, payload.options, payload.output_name)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- except ValueError as error:
- raise HTTPException(status_code=400, detail=str(error)) from error
- @app.get("/runs")
- def list_runs(limit: int = Query(20, ge=1, le=100)) -> dict:
- return {"items": runs.list(limit)}
- @app.get("/runs/{run_id}")
- def get_run(run_id: str) -> dict:
- try:
- return runs.get(run_id)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- # ---------------------------------------------------------------------- 工具
- class DataReadRequest(BaseModel):
- object_id: str
- table: str = "raw"
- columns: list[str] | None = None
- filters: list[dict] | None = None
- order_by: list[dict] | None = None
- limit: int | None = Field(default=None, gt=0)
- offset: int = 0
- aggregate: dict | None = None
- count_mode: str = "auto"
- cursor: str | None = None
- use_cache: bool = True
- #: 由 BFF 传入:把本次调用登记进数据面台账,call_id 与溯源令牌一致
- call_id: str = ""
- session_id: str = ""
- @app.post("/tools/data_read")
- def tool_data_read(payload: DataReadRequest) -> dict:
- try:
- result = read_rows(
- payload.object_id,
- table=payload.table,
- filters=payload.filters,
- columns=payload.columns,
- order_by=payload.order_by,
- limit=payload.limit,
- offset=payload.offset,
- aggregate=payload.aggregate,
- count_mode=payload.count_mode,
- cursor=payload.cursor,
- use_cache=payload.use_cache,
- )
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- except QueryTimeout as error:
- raise HTTPException(status_code=408, detail=str(error)) from error
- except ReadError as error:
- raise HTTPException(status_code=400, detail=str(error)) from error
- if payload.call_id:
- # 数据面台账:调用级记录 + 行级索引,供离线审计和"这个数从哪来"
- ledger.record(
- payload.call_id,
- "data_read",
- {
- "object_id": payload.object_id,
- "table": payload.table,
- "filters": payload.filters,
- "limit": payload.limit,
- "offset": payload.offset,
- "aggregate": payload.aggregate,
- "columns": payload.columns,
- "order_by": payload.order_by,
- "count_mode": payload.count_mode,
- "cursor": payload.cursor,
- "query_version": result.get("query_version"),
- },
- {
- "columns": result["columns"],
- "total": result.get("total"),
- "returned": len(result["rows"]),
- },
- object_ids=[payload.object_id],
- session_id=payload.session_id,
- rows=[
- {
- "values": row["values"],
- "_trace": {
- "object_id": row["_trace"].get("object_id", payload.object_id),
- "table": row["_trace"].get("table", payload.table),
- "row_index": row["_trace"].get("row_index", -1),
- "row_hash": row["_trace"].get("row_hash", ""),
- },
- }
- for row in result["rows"]
- ],
- )
- return result
- class CalcRequest(BaseModel):
- op: str = "sum"
- inputs: list[dict] | None = None
- expression: str | None = None
- session_id: str = ""
- label: str = ""
- @app.post("/tools/calculator")
- def tool_calculator(payload: CalcRequest) -> dict:
- try:
- return calculate(payload.op, payload.inputs, payload.expression, session_id=payload.session_id, label=payload.label)
- except CalcError as error:
- raise HTTPException(status_code=400, detail=str(error)) from error
- class ProcessRequest(BaseModel):
- workspace_id: str
- input_object_id: str
- operation: str = "full_pipeline"
- params: dict = Field(default_factory=dict)
- output_filename: str | None = None
- call_id: str = ""
- session_id: str = ""
- @app.post("/tools/customs_process")
- def tool_customs_process(payload: ProcessRequest) -> dict:
- if payload.operation not in OPERATIONS:
- raise HTTPException(status_code=400, detail=f"operation 只支持 {OPERATIONS}")
- try:
- run = runs.run_sync(payload.workspace_id, payload.input_object_id, payload.operation, payload.params, payload.output_filename)
- except KeyError as error:
- raise HTTPException(status_code=404, detail=str(error)) from error
- if run["status"] != "succeeded":
- return JSONResponse(
- status_code=422,
- content={"detail": run.get("error") or "处理失败", "run": run},
- )
- item = store.get_object(run["object_id"])
- if payload.call_id:
- ledger.record(
- payload.call_id,
- "customs_process",
- {
- "workspace_id": payload.workspace_id,
- "input_object_id": payload.input_object_id,
- "operation": payload.operation,
- "params": payload.params,
- },
- {
- "run_id": run["id"],
- "object_id": item["id"],
- "sheet_count": len(item["tables"]),
- "rows": item["rows"],
- "lineage": item["meta"].get("lineage", {}),
- },
- object_ids=[payload.input_object_id, item["id"]],
- session_id=payload.session_id,
- )
- return {
- "run_id": run["id"],
- "object_id": item["id"],
- "name": item["name"],
- "sheets": item["tables"],
- "lineage": item["meta"].get("lineage", {}),
- "stats": item["meta"].get("stats", {}),
- "warnings": item["meta"].get("warnings", []),
- "files": _object_files(item),
- }
- def _object_files(item: dict) -> list[str]:
- base = Path(item["path"])
- directory = base.parent if base.is_file() else base
- files = []
- for path in sorted(directory.rglob("*")):
- if path.is_file() and path.suffix.lower() in {".xlsx", ".csv", ".json", ".sqlite"}:
- files.append(str(path.relative_to(directory)).replace("\\", "/"))
- return files[:200]
- # ---------------------------------------------------------------------- 台账
- @app.get("/ledger/calls")
- def ledger_calls(session_id: str = "") -> dict:
- return {"items": ledger.session_calls(session_id)}
- @app.get("/ledger/value")
- def ledger_value(value: str, session_id: str = "") -> dict:
- return {"items": ledger.find_value(value, session_id)}
- @app.get("/ledger/stats")
- def ledger_stats() -> dict:
- return ledger.stats()
- @app.get("/ledger/call/{call_id}")
- def ledger_call(call_id: str) -> dict:
- item = ledger.get(call_id)
- if not item:
- raise HTTPException(status_code=404, detail="调用不存在")
- return item
- @app.get("/ledger/export")
- def ledger_export() -> dict:
- """导出 JSONL 台账(离线审计)。"""
- if not settings.ledger_path.exists():
- return {"items": []}
- lines = settings.ledger_path.read_text(encoding="utf-8").strip().splitlines()
- return {"items": [json.loads(line) for line in lines if line.strip()]}
- @app.exception_handler(Exception)
- def unhandled_error(_request, error: Exception): # noqa: ANN001
- return JSONResponse(status_code=500, content={"detail": f"{type(error).__name__}: {error}"})
|