| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175 |
- """默认实现:把海关原始 CSV/XLSX 处理成 16 张模板表 + 补充表。
- 替换方式见 ``toolsvc/customs/README.md``;本文件只做编排,不含业务算法。
- """
- from __future__ import annotations
- import hashlib
- import json
- import time
- from datetime import datetime
- from pathlib import Path
- from .contract import CustomsProcessRequest, CustomsProcessResult, ProgressCallback, ProcessorError, SheetInfo
- from .engine.rules import Rules, MEMORY_HS
- from .engine.query_summaries import build_query_summaries
- from .engine.storage import sha256_file, DIMENSIONS, connect, ingest
- from .engine import reports as reports_module
- ENGINE_VERSION = "2.2.0"
- DEFAULT_RULES = Path(__file__).with_name("rules")
- def _sha256_text(text: str) -> str:
- return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
- def _params_hash(payload: dict) -> str:
- return _sha256_text(json.dumps(payload, sort_keys=True, ensure_ascii=False))
- class DefaultCustomsProcessor:
- """参考实现。类属性 ``name``/``version`` 会写进血缘信息。"""
- name = "customs-default"
- version = ENGINE_VERSION
- def process(self, request: CustomsProcessRequest, progress: ProgressCallback | None = None):
- request.validate()
- report = progress or (lambda message, fraction: None)
- started = time.perf_counter()
- output_dir = Path(request.output_dir)
- output_dir.mkdir(parents=True, exist_ok=True)
- rules_dir = Path(request.rules_dir) if request.rules_dir else DEFAULT_RULES
- input_path = Path(request.input_path)
- if not input_path.exists():
- raise ProcessorError(f"输入文件不存在: {input_path}", code="INPUT_NOT_FOUND")
- options = request.options
- config = {
- **options.to_dict(),
- "input_csv": str(input_path),
- "mapping_file": str(rules_dir / "数据映射规则.xlsx"),
- "field_file": str(rules_dir / "进出口数据字段需求20260819.xlsx"),
- "template_file": str(rules_dir / "统计类数据输出格式.xlsx"),
- }
- report("装载规则库…", 0.03)
- try:
- rules = Rules(config["mapping_file"], config["field_file"], options.country_code_system)
- except Exception as error: # noqa: BLE001 - 规则文件损坏要给出可读原因
- raise ProcessorError(f"规则库加载失败: {error}", code="RULES_FAILED") from error
- run_config = dict(config)
- (output_dir / "run_config.json").write_text(json.dumps(run_config, ensure_ascii=False, indent=2), encoding="utf-8")
- report("读取原始数据…", 0.05)
- try:
- database, stats = ingest(config, rules, output_dir, progress)
- except Exception as error: # noqa: BLE001
- raise ProcessorError(f"数据入库失败: {error}", code="INGEST_FAILED") from error
- warnings: list[str] = []
- try:
- if request.operation == "custom_aggregate":
- sheets = self._custom_aggregate(database, options, output_dir, report)
- xlsx_path = output_dir / "统计结果.xlsx"
- sheets_written = sheets
- else:
- reporter = reports_module.Reports(database, config, stats, rules, output_dir, progress)
- xlsx_path = reporter.run()
- sheets_written = [
- SheetInfo(
- sheet=item["sheet"],
- kind=item["kind"],
- rows=reporter.output.sheets[item["sheet"]]["count"],
- columns=item["columns"],
- table=reporter.output.sheets[item["sheet"]]["table"],
- )
- for item in reporter.catalog
- ]
- report("构建查询专用叶子汇总…", 0.98)
- sheets_written.extend(build_query_summaries(database, output_dir))
- if stats["included"] == 0:
- warnings.append("没有任何记录通过范围校验,结果表全部为空;请检查 HS 编码、进出口方向与年份。")
- if stats["alerts"]:
- warnings.append(f"存在 {stats['alerts']} 条数值/范围异常记录,详见 issues.csv。")
- except ProcessorError:
- raise
- except Exception as error: # noqa: BLE001
- raise ProcessorError(f"报表生成失败: {error}", code="REPORT_FAILED") from error
- finally:
- database.close()
- stats["seconds"] = round(time.perf_counter() - started, 3)
- stats["engine"] = {"name": self.name, "version": self.version}
- (output_dir / "run_summary.json").write_text(json.dumps(stats, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
- rules_hash = _params_hash({name: sha256_file(config[name]) for name in ("mapping_file", "field_file", "template_file")})
- params_hash = _params_hash({"operation": request.operation, **options.to_dict()})
- output_hash = _params_hash({"input": stats.get("sha256"), "rules": rules_hash, "params": params_hash,
- "engine": self.version, "sqlite_sha256": sha256_file(output_dir / "stat.sqlite")})
- lineage = {
- "engine": {"name": self.name, "version": self.version},
- "input_sha256": stats.get("sha256"),
- "input_path": str(input_path),
- "params_hash": params_hash,
- "rules_hash": rules_hash,
- "processing_options": options.to_dict(),
- "hs_scope": sorted(set(rules.cap) | MEMORY_HS),
- "output_hash": output_hash,
- "started_at": datetime.now().isoformat(timespec="seconds"),
- "sheet_count": len(sheets_written),
- }
- report("处理完成", 1.0)
- return CustomsProcessResult(
- output_dir=str(output_dir),
- xlsx_path=str(xlsx_path),
- sqlite_path=str(output_dir / "stat.sqlite"),
- sheets=sheets_written,
- stats=stats,
- lineage=lineage,
- warnings=warnings,
- )
- # ------------------------------------------------------------------ extra op
- def _custom_aggregate(self, database, options, output_dir: Path, report) -> list[SheetInfo]:
- """自定义分组聚合:直接对入库后的 fact 表做 group by。"""
- from .engine.excel import Output
- from .engine.numbers import number
- group_by = [item for item in options.group_by if item in DIMENSIONS]
- if not group_by:
- raise ProcessorError("custom_aggregate 需要至少一个合法 group_by 维度", code="BAD_OPTIONS")
- filters = {key: value for key, value in options.filters.items() if key in DIMENSIONS}
- clause = " AND ".join(f"{key}=?" for key in filters) or "1=1"
- columns = ",".join(group_by)
- sql = (
- f'SELECT {columns},dsum(amount) amount,dsum(quantity) quantity,COUNT(*) n '
- f'FROM fact WHERE {clause} GROUP BY {columns} ORDER BY CAST(amount AS REAL) DESC'
- )
- report("执行自定义聚合…", 0.8)
- output = Output(output_dir)
- headers = group_by + ["实际金额(美元)", "成交数量", "记录数", "平均单价(美元/成交数量单位)"]
- output.sheet("自定义聚合", headers)
- for row in database.execute(sql, list(filters.values())):
- amount = number(row["amount"])
- quantity = number(row["quantity"])
- output.row(
- "自定义聚合",
- [row[key] for key in group_by]
- + [amount, quantity, row["n"], (amount / quantity) if amount is not None and quantity else None],
- )
- output.save()
- output.close()
- return [
- SheetInfo(
- sheet="自定义聚合",
- kind="custom",
- rows=output.sheets["自定义聚合"]["count"],
- columns=headers,
- table="s_自定义聚合",
- )
- ]
|