| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137 |
- """数据处理工具的对外契约(**替换点**)。
- 平台其余部分只依赖本文件里的数据类型与 ``CustomsProcessor`` 协议,
- 不 import ``engine`` 下的任何实现。要整体替换数据处理工具,只需:
- 1. 实现 ``CustomsProcessor.process(request, progress) -> CustomsProcessResult``;
- 2. 在 ``toolsvc.customs.factory.load_processor()`` 里换掉默认实现,或设置环境变量
- ``CUSTOMS_PROCESSOR`` 指向你的新类(``模块路径:类名``)。
- 前端直接调用的 ``POST /api/process`` 与 Agent 的 ``customs_process`` 工具
- 都走同一个协议,因此替换后两端同时生效,不需改前端。
- """
- from __future__ import annotations
- from dataclasses import dataclass, field
- from typing import Any, Callable, Protocol
- #: 进度回调:(阶段说明, 0~1 的完成度)
- ProgressCallback = Callable[[str, float], None]
- #: 支持的 operation
- OPERATIONS = ("full_pipeline", "custom_aggregate")
- @dataclass
- class CustomsOptions:
- """处理参数。默认值即《0909 需求问题整理与回复》的推荐口径。"""
- start_year: int = 2025
- end_year: int = 2026
- cutoff_date: str | None = None
- directions: list[str] = field(default_factory=lambda: ["I"])
- encoding: str = "auto"
- delimiter: str = "auto"
- country_code_system: str = "ISO"
- confirmed_complete_years: list[int] = field(default_factory=list)
- allow_partial_year_basket: bool = True
- demo: bool = False
- #: operation=custom_aggregate 时生效
- group_by: list[str] = field(default_factory=list)
- filters: dict[str, str] = field(default_factory=dict)
- def validate(self) -> None:
- if not (1900 <= int(self.start_year) <= int(self.end_year) <= 9998):
- raise ValueError("无效年份范围")
- if self.country_code_system not in ("ISO", "GEC"):
- raise ValueError("country_code_system 只支持 ISO/GEC")
- if not self.directions or set(self.directions) - {"I", "E"}:
- raise ValueError("directions 只支持 I / E")
- if self.delimiter not in ("auto", ",", "\t", ";", "|"):
- raise ValueError("只支持 , \\t ; | 四种分隔符")
- def to_dict(self) -> dict[str, Any]:
- return {
- "start_year": self.start_year,
- "end_year": self.end_year,
- "cutoff_date": self.cutoff_date,
- "directions": list(self.directions),
- "encoding": self.encoding,
- "delimiter": self.delimiter,
- "country_code_system": self.country_code_system,
- "confirmed_complete_years": list(self.confirmed_complete_years),
- "allow_partial_year_basket": self.allow_partial_year_basket,
- "demo": self.demo,
- "group_by": list(self.group_by),
- "filters": dict(self.filters),
- }
- @dataclass
- class CustomsProcessRequest:
- """一次处理请求。"""
- input_path: str
- output_dir: str
- operation: str = "full_pipeline"
- options: CustomsOptions = field(default_factory=CustomsOptions)
- #: 名称 → 绝对路径;允许调用方覆盖规则文件(默认用包内 rules/)
- rules_dir: str | None = None
- def validate(self) -> None:
- if self.operation not in OPERATIONS:
- raise ValueError(f"operation 只支持 {OPERATIONS}")
- self.options.validate()
- @dataclass
- class SheetInfo:
- sheet: str
- kind: str
- rows: int
- columns: list[str]
- table: str
- @dataclass
- class CustomsProcessResult:
- """处理结果。所有对外字段都是可序列化的基础类型。"""
- output_dir: str
- xlsx_path: str
- sqlite_path: str
- sheets: list[SheetInfo] = field(default_factory=list)
- stats: dict[str, Any] = field(default_factory=dict)
- lineage: dict[str, Any] = field(default_factory=dict)
- warnings: list[str] = field(default_factory=list)
- def to_dict(self) -> dict[str, Any]:
- return {
- "output_dir": self.output_dir,
- "xlsx_path": self.xlsx_path,
- "sqlite_path": self.sqlite_path,
- "sheets": [sheet.__dict__ for sheet in self.sheets],
- "stats": self.stats,
- "lineage": self.lineage,
- "warnings": self.warnings,
- }
- class CustomsProcessor(Protocol):
- """数据处理工具必须实现的协议。"""
- name: str
- version: str
- def process(self, request: CustomsProcessRequest, progress: ProgressCallback | None = None) -> CustomsProcessResult:
- ...
- class ProcessorError(RuntimeError):
- """处理失败时抛出,service 层转成结构化错误返回给前端/Agent。"""
- def __init__(self, message: str, *, code: str = "PROCESS_FAILED", detail: dict | None = None) -> None:
- super().__init__(message)
- self.code = code
- self.detail = detail or {}
|