| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194 |
- """处理任务(run):后台线程执行 ``customs_process``,前端轮询进度。
- 前端数据处理工作区与 Agent 工具共享同一套 run:
- * 前端:``POST /runs`` 起任务 → ``GET /runs/{id}`` 轮询 → 成功后拿到 ``object_id``;
- * Agent:``POST /tools/customs_process`` 同步等待同一个处理器返回结果。
- 同步接口也走这里,只是内部用 ``wait=True``。
- """
- from __future__ import annotations
- import json
- import threading
- import traceback
- from datetime import datetime, timezone
- from pathlib import Path
- from .config import settings
- from .customs import CustomsOptions, CustomsProcessRequest, ProcessorError, load_processor
- from .db import database
- from .objects import new_id, store
- def now() -> str:
- return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
- def build_options(payload: dict | None) -> CustomsOptions:
- payload = payload or {}
- allowed = {
- "start_year",
- "end_year",
- "cutoff_date",
- "directions",
- "encoding",
- "delimiter",
- "country_code_system",
- "confirmed_complete_years",
- "allow_partial_year_basket",
- "demo",
- "group_by",
- "filters",
- }
- data = {key: value for key, value in payload.items() if key in allowed and value is not None}
- return CustomsOptions(**data)
- class RunManager:
- """run 的创建、执行与查询。"""
- def __init__(self) -> None:
- settings.ensure()
- self._locks: dict[str, threading.Lock] = {}
- self._lock = threading.Lock()
- # ------------------------------------------------------------------ create
- def create(
- self,
- workspace_id: str,
- input_object_id: str,
- operation: str = "full_pipeline",
- options: dict | None = None,
- output_name: str | None = None,
- ) -> dict:
- source = store.get_object(input_object_id)
- if source["workspace_id"] != workspace_id:
- raise ValueError("输入对象不属于该工作区")
- run_id = new_id("run")
- run_dir = settings.runs_dir / run_id
- run_dir.mkdir(parents=True, exist_ok=True)
- with database.transaction() as connection:
- connection.execute(
- "INSERT INTO runs(id,object_id,input_object,status,operation,params_json,progress,message,started_at) "
- "VALUES(?,?,?,?,?,?,?,?,?)",
- (
- run_id,
- None,
- input_object_id,
- "queued",
- operation,
- json.dumps(options or {}, ensure_ascii=False),
- 0.0,
- "排队中",
- now(),
- ),
- )
- with self._lock:
- self._locks[run_id] = threading.Lock()
- thread = threading.Thread(
- target=self._execute,
- args=(run_id, workspace_id, input_object_id, operation, options or {}, output_name, run_dir),
- name=f"run-{run_id}",
- daemon=True,
- )
- thread.start()
- return self.get(run_id)
- # ------------------------------------------------------------------ execute
- def _execute(self, run_id: str, workspace_id: str, input_object_id: str, operation: str, options: dict, output_name: str | None, run_dir: Path) -> None:
- self._update(run_id, status="running", message="开始处理", progress=0.01)
- processor = load_processor()
- last = {"fraction": -1.0}
- def progress(message: str, fraction: float) -> None:
- if fraction - last["fraction"] >= 0.01 or fraction >= 0.999:
- last["fraction"] = fraction
- self._update(run_id, message=message, progress=round(fraction, 4))
- try:
- source = store.get_object(input_object_id)
- request = CustomsProcessRequest(
- input_path=source["path"],
- output_dir=str(run_dir),
- operation=operation,
- options=build_options(options),
- )
- result = processor.process(request, progress)
- self._update(run_id, message="登记结果对象…", progress=0.99)
- name = output_name or f"{Path(source['name']).stem}_统计结果.xlsx"
- item = store.register_processed(workspace_id, input_object_id, name, run_dir, result.to_dict())
- self._update(
- run_id,
- status="succeeded",
- object_id=item["id"],
- message="处理完成",
- progress=1.0,
- finished=True,
- )
- except ProcessorError as error:
- self._update(run_id, status="failed", error=f"{error.code}: {error}", message=str(error), progress=1.0, finished=True)
- except Exception as error: # noqa: BLE001
- detail = traceback.format_exc(limit=3)
- self._update(run_id, status="failed", error=f"{type(error).__name__}: {error}\n{detail}", message="处理失败", progress=1.0, finished=True)
- finally:
- with self._lock:
- self._locks.pop(run_id, None)
- def _update(self, run_id: str, **fields) -> None:
- allowed = {"status", "object_id", "progress", "message", "error"}
- assignments = []
- params: list = []
- for key, value in fields.items():
- if key in allowed:
- assignments.append(f"{key}=?")
- params.append(value)
- if fields.get("finished"):
- assignments.append("finished_at=?")
- params.append(now())
- if not assignments:
- return
- params.append(run_id)
- with database.transaction() as connection:
- connection.execute(f"UPDATE runs SET {', '.join(assignments)} WHERE id=?", params)
- # ------------------------------------------------------------------ query
- def get(self, run_id: str) -> dict:
- with database.connect() as connection:
- row = connection.execute("SELECT * FROM runs WHERE id=?", (run_id,)).fetchone()
- if not row:
- raise KeyError(f"任务不存在: {run_id}")
- item = dict(row)
- item["params"] = json.loads(item.pop("params_json") or "{}")
- return item
- def list(self, limit: int = 20) -> list[dict]:
- with database.connect() as connection:
- rows = connection.execute("SELECT id FROM runs ORDER BY started_at DESC LIMIT ?", (limit,)).fetchall()
- return [self.get(row["id"]) for row in rows]
- def run_sync(
- self,
- workspace_id: str,
- input_object_id: str,
- operation: str = "full_pipeline",
- options: dict | None = None,
- output_name: str | None = None,
- timeout: float = 1800,
- ) -> dict:
- """同步执行(Agent 工具用):阻塞到结束,返回 run 记录。"""
- run = self.create(workspace_id, input_object_id, operation, options, output_name)
- deadline = threading.Event()
- waited = 0.0
- while waited < timeout:
- current = self.get(run["id"])
- if current["status"] in ("succeeded", "failed"):
- return current
- deadline.wait(0.25)
- waited += 0.25
- return self.get(run["id"])
- runs = RunManager()
|