"""处理任务(run):后台线程执行 ``customs_process``,前端轮询进度。 前端数据处理工作区与 Agent 工具共享同一套 run: * 前端:``POST /runs`` 起任务 → ``GET /runs/{id}`` 轮询 → 成功后拿到 ``object_id``; * Agent:``POST /tools/customs_process`` 同步等待同一个处理器返回结果。 同步接口也走这里,只是内部用 ``wait=True``。 """ from __future__ import annotations import json import threading import traceback from datetime import datetime, timezone from pathlib import Path from .config import settings from .customs import CustomsOptions, CustomsProcessRequest, ProcessorError, load_processor from .db import database from .objects import new_id, store def now() -> str: return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") def build_options(payload: dict | None) -> CustomsOptions: payload = payload or {} allowed = { "start_year", "end_year", "cutoff_date", "directions", "encoding", "delimiter", "country_code_system", "confirmed_complete_years", "allow_partial_year_basket", "demo", "group_by", "filters", } data = {key: value for key, value in payload.items() if key in allowed and value is not None} return CustomsOptions(**data) class RunManager: """run 的创建、执行与查询。""" def __init__(self) -> None: settings.ensure() self._locks: dict[str, threading.Lock] = {} self._lock = threading.Lock() # ------------------------------------------------------------------ create def create( self, workspace_id: str, input_object_id: str, operation: str = "full_pipeline", options: dict | None = None, output_name: str | None = None, ) -> dict: source = store.get_object(input_object_id) if source["workspace_id"] != workspace_id: raise ValueError("输入对象不属于该工作区") run_id = new_id("run") run_dir = settings.runs_dir / run_id run_dir.mkdir(parents=True, exist_ok=True) with database.transaction() as connection: connection.execute( "INSERT INTO runs(id,object_id,input_object,status,operation,params_json,progress,message,started_at) " "VALUES(?,?,?,?,?,?,?,?,?)", ( run_id, None, input_object_id, "queued", operation, json.dumps(options or {}, ensure_ascii=False), 0.0, "排队中", now(), ), ) with self._lock: self._locks[run_id] = threading.Lock() thread = threading.Thread( target=self._execute, args=(run_id, workspace_id, input_object_id, operation, options or {}, output_name, run_dir), name=f"run-{run_id}", daemon=True, ) thread.start() return self.get(run_id) # ------------------------------------------------------------------ execute def _execute(self, run_id: str, workspace_id: str, input_object_id: str, operation: str, options: dict, output_name: str | None, run_dir: Path) -> None: self._update(run_id, status="running", message="开始处理", progress=0.01) processor = load_processor() last = {"fraction": -1.0} def progress(message: str, fraction: float) -> None: if fraction - last["fraction"] >= 0.01 or fraction >= 0.999: last["fraction"] = fraction self._update(run_id, message=message, progress=round(fraction, 4)) try: source = store.get_object(input_object_id) request = CustomsProcessRequest( input_path=source["path"], output_dir=str(run_dir), operation=operation, options=build_options(options), ) result = processor.process(request, progress) self._update(run_id, message="登记结果对象…", progress=0.99) name = output_name or f"{Path(source['name']).stem}_统计结果.xlsx" item = store.register_processed(workspace_id, input_object_id, name, run_dir, result.to_dict()) self._update( run_id, status="succeeded", object_id=item["id"], message="处理完成", progress=1.0, finished=True, ) except ProcessorError as error: self._update(run_id, status="failed", error=f"{error.code}: {error}", message=str(error), progress=1.0, finished=True) except Exception as error: # noqa: BLE001 detail = traceback.format_exc(limit=3) self._update(run_id, status="failed", error=f"{type(error).__name__}: {error}\n{detail}", message="处理失败", progress=1.0, finished=True) finally: with self._lock: self._locks.pop(run_id, None) def _update(self, run_id: str, **fields) -> None: allowed = {"status", "object_id", "progress", "message", "error"} assignments = [] params: list = [] for key, value in fields.items(): if key in allowed: assignments.append(f"{key}=?") params.append(value) if fields.get("finished"): assignments.append("finished_at=?") params.append(now()) if not assignments: return params.append(run_id) with database.transaction() as connection: connection.execute(f"UPDATE runs SET {', '.join(assignments)} WHERE id=?", params) # ------------------------------------------------------------------ query def get(self, run_id: str) -> dict: with database.connect() as connection: row = connection.execute("SELECT * FROM runs WHERE id=?", (run_id,)).fetchone() if not row: raise KeyError(f"任务不存在: {run_id}") item = dict(row) item["params"] = json.loads(item.pop("params_json") or "{}") return item def list(self, limit: int = 20) -> list[dict]: with database.connect() as connection: rows = connection.execute("SELECT id FROM runs ORDER BY started_at DESC LIMIT ?", (limit,)).fetchall() return [self.get(row["id"]) for row in rows] def run_sync( self, workspace_id: str, input_object_id: str, operation: str = "full_pipeline", options: dict | None = None, output_name: str | None = None, timeout: float = 1800, ) -> dict: """同步执行(Agent 工具用):阻塞到结束,返回 run 记录。""" run = self.create(workspace_id, input_object_id, operation, options, output_name) deadline = threading.Event() waited = 0.0 while waited < timeout: current = self.get(run["id"]) if current["status"] in ("succeeded", "failed"): return current deadline.wait(0.25) waited += 0.25 return self.get(run["id"]) runs = RunManager()