"""默认实现:把海关原始 CSV/XLSX 处理成 16 张模板表 + 补充表。 替换方式见 ``toolsvc/customs/README.md``;本文件只做编排,不含业务算法。 """ from __future__ import annotations import hashlib import json import time from datetime import datetime from pathlib import Path from .contract import CustomsProcessRequest, CustomsProcessResult, ProgressCallback, ProcessorError, SheetInfo from .engine.rules import Rules, MEMORY_HS from .engine.query_summaries import build_query_summaries from .engine.storage import sha256_file, DIMENSIONS, connect, ingest from .engine import reports as reports_module ENGINE_VERSION = "2.2.0" DEFAULT_RULES = Path(__file__).with_name("rules") def _sha256_text(text: str) -> str: return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() def _params_hash(payload: dict) -> str: return _sha256_text(json.dumps(payload, sort_keys=True, ensure_ascii=False)) class DefaultCustomsProcessor: """参考实现。类属性 ``name``/``version`` 会写进血缘信息。""" name = "customs-default" version = ENGINE_VERSION def process(self, request: CustomsProcessRequest, progress: ProgressCallback | None = None): request.validate() report = progress or (lambda message, fraction: None) started = time.perf_counter() output_dir = Path(request.output_dir) output_dir.mkdir(parents=True, exist_ok=True) rules_dir = Path(request.rules_dir) if request.rules_dir else DEFAULT_RULES input_path = Path(request.input_path) if not input_path.exists(): raise ProcessorError(f"输入文件不存在: {input_path}", code="INPUT_NOT_FOUND") options = request.options config = { **options.to_dict(), "input_csv": str(input_path), "mapping_file": str(rules_dir / "数据映射规则.xlsx"), "field_file": str(rules_dir / "进出口数据字段需求20260819.xlsx"), "template_file": str(rules_dir / "统计类数据输出格式.xlsx"), } report("装载规则库…", 0.03) try: rules = Rules(config["mapping_file"], config["field_file"], options.country_code_system) except Exception as error: # noqa: BLE001 - 规则文件损坏要给出可读原因 raise ProcessorError(f"规则库加载失败: {error}", code="RULES_FAILED") from error run_config = dict(config) (output_dir / "run_config.json").write_text(json.dumps(run_config, ensure_ascii=False, indent=2), encoding="utf-8") report("读取原始数据…", 0.05) try: database, stats = ingest(config, rules, output_dir, progress) except Exception as error: # noqa: BLE001 raise ProcessorError(f"数据入库失败: {error}", code="INGEST_FAILED") from error warnings: list[str] = [] try: if request.operation == "custom_aggregate": sheets = self._custom_aggregate(database, options, output_dir, report) xlsx_path = output_dir / "统计结果.xlsx" sheets_written = sheets else: reporter = reports_module.Reports(database, config, stats, rules, output_dir, progress) xlsx_path = reporter.run() sheets_written = [ SheetInfo( sheet=item["sheet"], kind=item["kind"], rows=reporter.output.sheets[item["sheet"]]["count"], columns=item["columns"], table=reporter.output.sheets[item["sheet"]]["table"], ) for item in reporter.catalog ] report("构建查询专用叶子汇总…", 0.98) sheets_written.extend(build_query_summaries(database, output_dir)) if stats["included"] == 0: warnings.append("没有任何记录通过范围校验,结果表全部为空;请检查 HS 编码、进出口方向与年份。") if stats["alerts"]: warnings.append(f"存在 {stats['alerts']} 条数值/范围异常记录,详见 issues.csv。") except ProcessorError: raise except Exception as error: # noqa: BLE001 raise ProcessorError(f"报表生成失败: {error}", code="REPORT_FAILED") from error finally: database.close() stats["seconds"] = round(time.perf_counter() - started, 3) stats["engine"] = {"name": self.name, "version": self.version} (output_dir / "run_summary.json").write_text(json.dumps(stats, ensure_ascii=False, indent=2, default=str), encoding="utf-8") rules_hash = _params_hash({name: sha256_file(config[name]) for name in ("mapping_file", "field_file", "template_file")}) params_hash = _params_hash({"operation": request.operation, **options.to_dict()}) output_hash = _params_hash({"input": stats.get("sha256"), "rules": rules_hash, "params": params_hash, "engine": self.version, "sqlite_sha256": sha256_file(output_dir / "stat.sqlite")}) lineage = { "engine": {"name": self.name, "version": self.version}, "input_sha256": stats.get("sha256"), "input_path": str(input_path), "params_hash": params_hash, "rules_hash": rules_hash, "processing_options": options.to_dict(), "hs_scope": sorted(set(rules.cap) | MEMORY_HS), "output_hash": output_hash, "started_at": datetime.now().isoformat(timespec="seconds"), "sheet_count": len(sheets_written), } report("处理完成", 1.0) return CustomsProcessResult( output_dir=str(output_dir), xlsx_path=str(xlsx_path), sqlite_path=str(output_dir / "stat.sqlite"), sheets=sheets_written, stats=stats, lineage=lineage, warnings=warnings, ) # ------------------------------------------------------------------ extra op def _custom_aggregate(self, database, options, output_dir: Path, report) -> list[SheetInfo]: """自定义分组聚合:直接对入库后的 fact 表做 group by。""" from .engine.excel import Output from .engine.numbers import number group_by = [item for item in options.group_by if item in DIMENSIONS] if not group_by: raise ProcessorError("custom_aggregate 需要至少一个合法 group_by 维度", code="BAD_OPTIONS") filters = {key: value for key, value in options.filters.items() if key in DIMENSIONS} clause = " AND ".join(f"{key}=?" for key in filters) or "1=1" columns = ",".join(group_by) sql = ( f'SELECT {columns},dsum(amount) amount,dsum(quantity) quantity,COUNT(*) n ' f'FROM fact WHERE {clause} GROUP BY {columns} ORDER BY CAST(amount AS REAL) DESC' ) report("执行自定义聚合…", 0.8) output = Output(output_dir) headers = group_by + ["实际金额(美元)", "成交数量", "记录数", "平均单价(美元/成交数量单位)"] output.sheet("自定义聚合", headers) for row in database.execute(sql, list(filters.values())): amount = number(row["amount"]) quantity = number(row["quantity"]) output.row( "自定义聚合", [row[key] for key in group_by] + [amount, quantity, row["n"], (amount / quantity) if amount is not None and quantity else None], ) output.save() output.close() return [ SheetInfo( sheet="自定义聚合", kind="custom", rows=output.sheets["自定义聚合"]["count"], columns=headers, table="s_自定义聚合", ) ]