pipeline.py 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. """默认实现:把海关原始 CSV/XLSX 处理成 16 张模板表 + 补充表。
  2. 替换方式见 ``toolsvc/customs/README.md``;本文件只做编排,不含业务算法。
  3. """
  4. from __future__ import annotations
  5. import hashlib
  6. import json
  7. import time
  8. from datetime import datetime
  9. from pathlib import Path
  10. from .contract import CustomsProcessRequest, CustomsProcessResult, ProgressCallback, ProcessorError, SheetInfo
  11. from .engine.rules import Rules, MEMORY_HS
  12. from .engine.query_summaries import build_query_summaries
  13. from .engine.storage import sha256_file, DIMENSIONS, connect, ingest
  14. from .engine import reports as reports_module
  15. ENGINE_VERSION = "2.2.0"
  16. DEFAULT_RULES = Path(__file__).with_name("rules")
  17. def _sha256_text(text: str) -> str:
  18. return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
  19. def _params_hash(payload: dict) -> str:
  20. return _sha256_text(json.dumps(payload, sort_keys=True, ensure_ascii=False))
  21. class DefaultCustomsProcessor:
  22. """参考实现。类属性 ``name``/``version`` 会写进血缘信息。"""
  23. name = "customs-default"
  24. version = ENGINE_VERSION
  25. def process(self, request: CustomsProcessRequest, progress: ProgressCallback | None = None):
  26. request.validate()
  27. report = progress or (lambda message, fraction: None)
  28. started = time.perf_counter()
  29. output_dir = Path(request.output_dir)
  30. output_dir.mkdir(parents=True, exist_ok=True)
  31. rules_dir = Path(request.rules_dir) if request.rules_dir else DEFAULT_RULES
  32. input_path = Path(request.input_path)
  33. if not input_path.exists():
  34. raise ProcessorError(f"输入文件不存在: {input_path}", code="INPUT_NOT_FOUND")
  35. options = request.options
  36. config = {
  37. **options.to_dict(),
  38. "input_csv": str(input_path),
  39. "mapping_file": str(rules_dir / "数据映射规则.xlsx"),
  40. "field_file": str(rules_dir / "进出口数据字段需求20260819.xlsx"),
  41. "template_file": str(rules_dir / "统计类数据输出格式.xlsx"),
  42. }
  43. report("装载规则库…", 0.03)
  44. try:
  45. rules = Rules(config["mapping_file"], config["field_file"], options.country_code_system)
  46. except Exception as error: # noqa: BLE001 - 规则文件损坏要给出可读原因
  47. raise ProcessorError(f"规则库加载失败: {error}", code="RULES_FAILED") from error
  48. run_config = dict(config)
  49. (output_dir / "run_config.json").write_text(json.dumps(run_config, ensure_ascii=False, indent=2), encoding="utf-8")
  50. report("读取原始数据…", 0.05)
  51. try:
  52. database, stats = ingest(config, rules, output_dir, progress)
  53. except Exception as error: # noqa: BLE001
  54. raise ProcessorError(f"数据入库失败: {error}", code="INGEST_FAILED") from error
  55. warnings: list[str] = []
  56. try:
  57. if request.operation == "custom_aggregate":
  58. sheets = self._custom_aggregate(database, options, output_dir, report)
  59. xlsx_path = output_dir / "统计结果.xlsx"
  60. sheets_written = sheets
  61. else:
  62. reporter = reports_module.Reports(database, config, stats, rules, output_dir, progress)
  63. xlsx_path = reporter.run()
  64. sheets_written = [
  65. SheetInfo(
  66. sheet=item["sheet"],
  67. kind=item["kind"],
  68. rows=reporter.output.sheets[item["sheet"]]["count"],
  69. columns=item["columns"],
  70. table=reporter.output.sheets[item["sheet"]]["table"],
  71. )
  72. for item in reporter.catalog
  73. ]
  74. report("构建查询专用叶子汇总…", 0.98)
  75. sheets_written.extend(build_query_summaries(database, output_dir))
  76. if stats["included"] == 0:
  77. warnings.append("没有任何记录通过范围校验,结果表全部为空;请检查 HS 编码、进出口方向与年份。")
  78. if stats["alerts"]:
  79. warnings.append(f"存在 {stats['alerts']} 条数值/范围异常记录,详见 issues.csv。")
  80. except ProcessorError:
  81. raise
  82. except Exception as error: # noqa: BLE001
  83. raise ProcessorError(f"报表生成失败: {error}", code="REPORT_FAILED") from error
  84. finally:
  85. database.close()
  86. stats["seconds"] = round(time.perf_counter() - started, 3)
  87. stats["engine"] = {"name": self.name, "version": self.version}
  88. (output_dir / "run_summary.json").write_text(json.dumps(stats, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
  89. rules_hash = _params_hash({name: sha256_file(config[name]) for name in ("mapping_file", "field_file", "template_file")})
  90. params_hash = _params_hash({"operation": request.operation, **options.to_dict()})
  91. output_hash = _params_hash({"input": stats.get("sha256"), "rules": rules_hash, "params": params_hash,
  92. "engine": self.version, "sqlite_sha256": sha256_file(output_dir / "stat.sqlite")})
  93. lineage = {
  94. "engine": {"name": self.name, "version": self.version},
  95. "input_sha256": stats.get("sha256"),
  96. "input_path": str(input_path),
  97. "params_hash": params_hash,
  98. "rules_hash": rules_hash,
  99. "processing_options": options.to_dict(),
  100. "hs_scope": sorted(set(rules.cap) | MEMORY_HS),
  101. "output_hash": output_hash,
  102. "started_at": datetime.now().isoformat(timespec="seconds"),
  103. "sheet_count": len(sheets_written),
  104. }
  105. report("处理完成", 1.0)
  106. return CustomsProcessResult(
  107. output_dir=str(output_dir),
  108. xlsx_path=str(xlsx_path),
  109. sqlite_path=str(output_dir / "stat.sqlite"),
  110. sheets=sheets_written,
  111. stats=stats,
  112. lineage=lineage,
  113. warnings=warnings,
  114. )
  115. # ------------------------------------------------------------------ extra op
  116. def _custom_aggregate(self, database, options, output_dir: Path, report) -> list[SheetInfo]:
  117. """自定义分组聚合:直接对入库后的 fact 表做 group by。"""
  118. from .engine.excel import Output
  119. from .engine.numbers import number
  120. group_by = [item for item in options.group_by if item in DIMENSIONS]
  121. if not group_by:
  122. raise ProcessorError("custom_aggregate 需要至少一个合法 group_by 维度", code="BAD_OPTIONS")
  123. filters = {key: value for key, value in options.filters.items() if key in DIMENSIONS}
  124. clause = " AND ".join(f"{key}=?" for key in filters) or "1=1"
  125. columns = ",".join(group_by)
  126. sql = (
  127. f'SELECT {columns},dsum(amount) amount,dsum(quantity) quantity,COUNT(*) n '
  128. f'FROM fact WHERE {clause} GROUP BY {columns} ORDER BY CAST(amount AS REAL) DESC'
  129. )
  130. report("执行自定义聚合…", 0.8)
  131. output = Output(output_dir)
  132. headers = group_by + ["实际金额(美元)", "成交数量", "记录数", "平均单价(美元/成交数量单位)"]
  133. output.sheet("自定义聚合", headers)
  134. for row in database.execute(sql, list(filters.values())):
  135. amount = number(row["amount"])
  136. quantity = number(row["quantity"])
  137. output.row(
  138. "自定义聚合",
  139. [row[key] for key in group_by]
  140. + [amount, quantity, row["n"], (amount / quantity) if amount is not None and quantity else None],
  141. )
  142. output.save()
  143. output.close()
  144. return [
  145. SheetInfo(
  146. sheet="自定义聚合",
  147. kind="custom",
  148. rows=output.sheets["自定义聚合"]["count"],
  149. columns=headers,
  150. table="s_自定义聚合",
  151. )
  152. ]