runs.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194
  1. """处理任务(run):后台线程执行 ``customs_process``,前端轮询进度。
  2. 前端数据处理工作区与 Agent 工具共享同一套 run:
  3. * 前端:``POST /runs`` 起任务 → ``GET /runs/{id}`` 轮询 → 成功后拿到 ``object_id``;
  4. * Agent:``POST /tools/customs_process`` 同步等待同一个处理器返回结果。
  5. 同步接口也走这里,只是内部用 ``wait=True``。
  6. """
  7. from __future__ import annotations
  8. import json
  9. import threading
  10. import traceback
  11. from datetime import datetime, timezone
  12. from pathlib import Path
  13. from .config import settings
  14. from .customs import CustomsOptions, CustomsProcessRequest, ProcessorError, load_processor
  15. from .db import database
  16. from .objects import new_id, store
  17. def now() -> str:
  18. return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
  19. def build_options(payload: dict | None) -> CustomsOptions:
  20. payload = payload or {}
  21. allowed = {
  22. "start_year",
  23. "end_year",
  24. "cutoff_date",
  25. "directions",
  26. "encoding",
  27. "delimiter",
  28. "country_code_system",
  29. "confirmed_complete_years",
  30. "allow_partial_year_basket",
  31. "demo",
  32. "group_by",
  33. "filters",
  34. }
  35. data = {key: value for key, value in payload.items() if key in allowed and value is not None}
  36. return CustomsOptions(**data)
  37. class RunManager:
  38. """run 的创建、执行与查询。"""
  39. def __init__(self) -> None:
  40. settings.ensure()
  41. self._locks: dict[str, threading.Lock] = {}
  42. self._lock = threading.Lock()
  43. # ------------------------------------------------------------------ create
  44. def create(
  45. self,
  46. workspace_id: str,
  47. input_object_id: str,
  48. operation: str = "full_pipeline",
  49. options: dict | None = None,
  50. output_name: str | None = None,
  51. ) -> dict:
  52. source = store.get_object(input_object_id)
  53. if source["workspace_id"] != workspace_id:
  54. raise ValueError("输入对象不属于该工作区")
  55. run_id = new_id("run")
  56. run_dir = settings.runs_dir / run_id
  57. run_dir.mkdir(parents=True, exist_ok=True)
  58. with database.transaction() as connection:
  59. connection.execute(
  60. "INSERT INTO runs(id,object_id,input_object,status,operation,params_json,progress,message,started_at) "
  61. "VALUES(?,?,?,?,?,?,?,?,?)",
  62. (
  63. run_id,
  64. None,
  65. input_object_id,
  66. "queued",
  67. operation,
  68. json.dumps(options or {}, ensure_ascii=False),
  69. 0.0,
  70. "排队中",
  71. now(),
  72. ),
  73. )
  74. with self._lock:
  75. self._locks[run_id] = threading.Lock()
  76. thread = threading.Thread(
  77. target=self._execute,
  78. args=(run_id, workspace_id, input_object_id, operation, options or {}, output_name, run_dir),
  79. name=f"run-{run_id}",
  80. daemon=True,
  81. )
  82. thread.start()
  83. return self.get(run_id)
  84. # ------------------------------------------------------------------ execute
  85. def _execute(self, run_id: str, workspace_id: str, input_object_id: str, operation: str, options: dict, output_name: str | None, run_dir: Path) -> None:
  86. self._update(run_id, status="running", message="开始处理", progress=0.01)
  87. processor = load_processor()
  88. last = {"fraction": -1.0}
  89. def progress(message: str, fraction: float) -> None:
  90. if fraction - last["fraction"] >= 0.01 or fraction >= 0.999:
  91. last["fraction"] = fraction
  92. self._update(run_id, message=message, progress=round(fraction, 4))
  93. try:
  94. source = store.get_object(input_object_id)
  95. request = CustomsProcessRequest(
  96. input_path=source["path"],
  97. output_dir=str(run_dir),
  98. operation=operation,
  99. options=build_options(options),
  100. )
  101. result = processor.process(request, progress)
  102. self._update(run_id, message="登记结果对象…", progress=0.99)
  103. name = output_name or f"{Path(source['name']).stem}_统计结果.xlsx"
  104. item = store.register_processed(workspace_id, input_object_id, name, run_dir, result.to_dict())
  105. self._update(
  106. run_id,
  107. status="succeeded",
  108. object_id=item["id"],
  109. message="处理完成",
  110. progress=1.0,
  111. finished=True,
  112. )
  113. except ProcessorError as error:
  114. self._update(run_id, status="failed", error=f"{error.code}: {error}", message=str(error), progress=1.0, finished=True)
  115. except Exception as error: # noqa: BLE001
  116. detail = traceback.format_exc(limit=3)
  117. self._update(run_id, status="failed", error=f"{type(error).__name__}: {error}\n{detail}", message="处理失败", progress=1.0, finished=True)
  118. finally:
  119. with self._lock:
  120. self._locks.pop(run_id, None)
  121. def _update(self, run_id: str, **fields) -> None:
  122. allowed = {"status", "object_id", "progress", "message", "error"}
  123. assignments = []
  124. params: list = []
  125. for key, value in fields.items():
  126. if key in allowed:
  127. assignments.append(f"{key}=?")
  128. params.append(value)
  129. if fields.get("finished"):
  130. assignments.append("finished_at=?")
  131. params.append(now())
  132. if not assignments:
  133. return
  134. params.append(run_id)
  135. with database.transaction() as connection:
  136. connection.execute(f"UPDATE runs SET {', '.join(assignments)} WHERE id=?", params)
  137. # ------------------------------------------------------------------ query
  138. def get(self, run_id: str) -> dict:
  139. with database.connect() as connection:
  140. row = connection.execute("SELECT * FROM runs WHERE id=?", (run_id,)).fetchone()
  141. if not row:
  142. raise KeyError(f"任务不存在: {run_id}")
  143. item = dict(row)
  144. item["params"] = json.loads(item.pop("params_json") or "{}")
  145. return item
  146. def list(self, limit: int = 20) -> list[dict]:
  147. with database.connect() as connection:
  148. rows = connection.execute("SELECT id FROM runs ORDER BY started_at DESC LIMIT ?", (limit,)).fetchall()
  149. return [self.get(row["id"]) for row in rows]
  150. def run_sync(
  151. self,
  152. workspace_id: str,
  153. input_object_id: str,
  154. operation: str = "full_pipeline",
  155. options: dict | None = None,
  156. output_name: str | None = None,
  157. timeout: float = 1800,
  158. ) -> dict:
  159. """同步执行(Agent 工具用):阻塞到结束,返回 run 记录。"""
  160. run = self.create(workspace_id, input_object_id, operation, options, output_name)
  161. deadline = threading.Event()
  162. waited = 0.0
  163. while waited < timeout:
  164. current = self.get(run["id"])
  165. if current["status"] in ("succeeded", "failed"):
  166. return current
  167. deadline.wait(0.25)
  168. waited += 0.25
  169. return self.get(run["id"])
  170. runs = RunManager()