test_pipeline.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256
  1. """端到端口径测试:用样本数据跑完整流程,并对结果做业务断言。
  2. 预言机来自 ``scripts/gen_sample_data.py`` 写出的 ``*.expected.json``,
  3. 它只依赖“生成了哪些行”,不依赖解析/分类逻辑,因此能独立校验引擎。
  4. """
  5. from __future__ import annotations
  6. import sqlite3
  7. import pytest
  8. TEMPLATE_SHEETS = [
  9. "1.0-3.0_MLCC",
  10. "4.1_MLCC",
  11. "4.2.1_MLCC",
  12. "4.2.2_MLCC",
  13. "4.2.3 (1-4)_MLCC",
  14. "4.2.3 (5)_MLCC",
  15. "4.3_MLCC",
  16. "1.0-3.0_存储器",
  17. "5.1_存储器",
  18. "5.2.1&3&5_存储器",
  19. "5.2.2_存储器",
  20. "5.2.4_存储器",
  21. "5.2.6_存储器",
  22. "5.2.7 (1-4)_存储器",
  23. "5.2.7 (5)_存储器",
  24. "5.3_存储器",
  25. ]
  26. SUPPLEMENTARY_SHEETS = [
  27. "样本权重",
  28. "选样排除",
  29. "重点全量层级",
  30. "供应链条件占比",
  31. "周四归月映射",
  32. "数据质量摘要",
  33. "映射冲突",
  34. ]
  35. def table_for(sheet: str) -> str:
  36. import re
  37. cleaned = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "_", sheet).strip("_")
  38. return f"s_{cleaned}"
  39. @pytest.fixture(scope="module")
  40. def stat_db(processed):
  41. path = processed["result"].sqlite_path
  42. connection = sqlite3.connect(path)
  43. connection.row_factory = sqlite3.Row
  44. yield connection
  45. connection.close()
  46. def test_all_template_sheets_are_present(processed):
  47. names = [sheet.sheet for sheet in processed["result"].sheets]
  48. for sheet in TEMPLATE_SHEETS:
  49. assert sheet in names, f"缺少模板表 {sheet}"
  50. for sheet in SUPPLEMENTARY_SHEETS:
  51. assert sheet in names, f"缺少补充表 {sheet}"
  52. assert len(processed["result"].sheets) == 25
  53. def test_template_headers_keep_the_canonical_metric_names(stat_db):
  54. header_row = stat_db.execute(
  55. 'SELECT * FROM "s_1_0_3_0_MLCC" LIMIT 1'
  56. ).fetchone()
  57. columns = list(header_row.keys())
  58. for expected in ["实际金额(美元)", "成交数量", "平均单价(美元/成交数量单位)", "金额指数", "数量指数", "价格指数", "金额占比", "数量占比", "金额环比", "数量环比", "计算说明"]:
  59. assert expected in columns
  60. # 旧模板的“前五大型号”必须被纠正为“前十大型号”
  61. top_columns = stat_db.execute('SELECT * FROM "s_4_2_3_1_4_MLCC" LIMIT 1').fetchone().keys()
  62. assert "前十大型号" in top_columns
  63. assert "前五大型号" not in top_columns
  64. def test_row_scope_matches_expected_oracle(processed, sample_expected):
  65. stats = processed["result"].stats
  66. assert stats["total"] == sample_expected["rows"]
  67. assert stats["included"] == sample_expected["included"]
  68. assert stats["excluded"] == sample_expected["excluded"]
  69. assert stats["min_month"] == sample_expected["months"][0]
  70. assert stats["max_month"] == sample_expected["months"][1]
  71. def test_monthly_totals_match_expected_amount(processed, sample_expected):
  72. working = sqlite3.connect(processed["output"] / "working.sqlite")
  73. try:
  74. total = working.execute("SELECT SUM(CAST(amount AS REAL)) FROM monthly").fetchone()[0]
  75. quantity = working.execute("SELECT SUM(CAST(quantity AS REAL)) FROM monthly").fetchone()[0]
  76. finally:
  77. working.close()
  78. assert total == pytest.approx(sample_expected["amount"], rel=1e-9)
  79. assert quantity == pytest.approx(sample_expected["quantity"], rel=1e-9)
  80. def test_product_totals_split_by_first_level_category(stat_db, sample_expected):
  81. capacitor = stat_db.execute(
  82. f'SELECT SUM(CAST("_num_实际金额(美元)" AS REAL)) FROM "{table_for("4.1_MLCC")}"'
  83. ).fetchone()[0]
  84. memory = stat_db.execute(
  85. f'SELECT SUM(CAST("_num_实际金额(美元)" AS REAL)) FROM "{table_for("5.1_存储器")}"'
  86. ).fetchone()[0]
  87. assert capacitor == pytest.approx(sample_expected["amount_by_c1"]["电容器"], rel=1e-9)
  88. assert memory == pytest.approx(sample_expected["amount_by_c1"]["存储器"], rel=1e-9)
  89. def test_base_month_indices_are_exactly_1000(stat_db):
  90. rows = stat_db.execute(
  91. f'SELECT * FROM "{table_for("1.0-3.0_MLCC")}" WHERE "年度"="2025" AND "月度"="1"'
  92. ).fetchall()
  93. assert rows, "2025-01 应该有指数行"
  94. for row in rows:
  95. assert float(row["_num_金额指数"]) == pytest.approx(1000.0, abs=1e-6)
  96. assert float(row["_num_数量指数"]) == pytest.approx(1000.0, abs=1e-6)
  97. def test_price_index_at_base_month_is_1000_when_basket_complete(stat_db):
  98. """样本充足时,加权价格指数在基期必须刚好等于 1000。"""
  99. rows = stat_db.execute(
  100. f'SELECT * FROM "{table_for("1.0-3.0_MLCC")}" '
  101. 'WHERE "年度"="2025" AND "月度"="1" AND "基础类目" LIKE "%MLCC%"'
  102. ).fetchall()
  103. assert rows, "缺少 MLCC 基期行"
  104. price = rows[0]["_num_价格指数"]
  105. assert price is not None, f"价格指数不应为空:{dict(rows[0])}"
  106. assert float(price) == pytest.approx(1000.0, abs=1e-6)
  107. def test_price_basket_weights_sum_to_one(stat_db):
  108. rows = stat_db.execute(
  109. f'SELECT "基础类目","选样年度",SUM(CAST("_num_权重" AS REAL)) s,COUNT(*) n FROM "{table_for("样本权重")}" GROUP BY "基础类目","选样年度"'
  110. ).fetchall()
  111. assert rows
  112. for row in rows:
  113. assert row["s"] == pytest.approx(1.0, abs=1e-9), f"{row['基础类目']} {row['选样年度']} 权重合计 {row['s']}"
  114. assert row["n"] <= 100
  115. def test_price_basket_cap_is_respected(stat_db):
  116. rows = stat_db.execute(
  117. f'SELECT "基础类目","选样年度",MAX(CAST("_num_权重" AS REAL)) m,COUNT(*) n FROM "{table_for("样本权重")}" GROUP BY "基础类目","选样年度"'
  118. ).fetchall()
  119. for row in rows:
  120. if row["n"] >= 10:
  121. assert row["m"] <= 0.1 + 1e-12
  122. def test_annual_structure_shares_sum_to_one(stat_db):
  123. rows = stat_db.execute(
  124. f'SELECT "年度",SUM(CAST("_num_金额占比" AS REAL)) s FROM "{table_for("4.2.1_MLCC")}" GROUP BY "年度"'
  125. ).fetchall()
  126. assert rows
  127. for row in rows:
  128. assert row["s"] == pytest.approx(1.0, abs=1e-9)
  129. def test_nested_analysis_has_other_bucket_and_top10(stat_db):
  130. rows = stat_db.execute(
  131. f'SELECT "统计层级","应用领域标签","品牌国别","品牌","前十大型号" FROM "{table_for("4.2.3 (1-4)_MLCC")}"'
  132. ).fetchall()
  133. levels = {row["统计层级"] for row in rows}
  134. assert {"应用领域标签", "品牌国别", "品牌", "前十大型号"} <= levels
  135. flattened = [str(value) for row in rows for value in tuple(row)[1:] if value is not None]
  136. assert any("其他" in value for value in flattened), "缺少“其他”汇总桶"
  137. models = stat_db.execute(
  138. f"""SELECT DISTINCT "品牌","前十大型号" FROM "{table_for("4.2.3 (1-4)_MLCC")}" WHERE "统计层级"='前十大型号'"""
  139. ).fetchall()
  140. per_brand: dict[str, int] = {}
  141. for row in models:
  142. per_brand[row["品牌"]] = per_brand.get(row["品牌"], 0) + 1
  143. assert per_brand, "缺少型号层级"
  144. assert max(per_brand.values()) <= 11 # 10 个型号 + “其他(未入选合计)”
  145. def test_monthly_model_sheet_has_two_scope_versions(stat_db):
  146. rows = stat_db.execute(
  147. f'SELECT DISTINCT "统计范围" FROM "{table_for("4.2.3 (5)_MLCC")}"'
  148. ).fetchall()
  149. scopes = {row["统计范围"] for row in rows}
  150. assert "项目范围全部进口" in scopes
  151. assert "限定入选路径" in scopes
  152. def test_supply_chain_sheet_covers_both_port_granularities(stat_db):
  153. rows = stat_db.execute(
  154. f'SELECT DISTINCT "港口统计粒度" FROM "{table_for("4.3_MLCC")}"'
  155. ).fetchall()
  156. granularities = {row["港口统计粒度"] for row in rows}
  157. assert {"具体港口", "港口国家地区"} <= granularities
  158. def test_condition_share_sheet_has_two_denominator_modes(stat_db):
  159. rows = stat_db.execute(
  160. f'SELECT DISTINCT "分析方式" FROM "{table_for("供应链条件占比")}"'
  161. ).fetchall()
  162. modes = {row["分析方式"] for row in rows}
  163. assert {"完整前缀条件", "品牌国别条件"} <= modes
  164. def test_week_mapping_sheet_uses_iso_thursday_rule(stat_db):
  165. rows = stat_db.execute(
  166. f'SELECT COUNT(*) c FROM "{table_for("周四归月映射")}" WHERE "规则" LIKE "ISO%"'
  167. ).fetchone()
  168. assert rows["c"] > 100
  169. def test_quality_sheet_reports_required_metadata(stat_db):
  170. items = {
  171. row["项目"]: row["值"]
  172. for row in stat_db.execute(f'SELECT "项目","值" FROM "{table_for("数据质量摘要")}"')
  173. }
  174. for key in ["币种", "数量单位", "日期口径", "基期", "精度", "国家代码制"]:
  175. assert key in items
  176. assert items["币种"] == "USD"
  177. assert items["基期"] == "2025-01"
  178. def test_detail_csv_files_are_written(processed):
  179. output = processed["output"]
  180. for name in ["issues.csv", "audit.csv", "raw.csv", "monthly.csv", "解析明细.csv", "应用二轮变化.csv", "未识别及提取异常.csv"]:
  181. assert (output / name).exists(), f"缺少明细文件 {name}"
  182. def test_excel_workbook_contains_all_sheets(processed):
  183. from openpyxl import load_workbook
  184. workbook = load_workbook(processed["result"].xlsx_path, read_only=True)
  185. try:
  186. names = workbook.sheetnames
  187. finally:
  188. workbook.close()
  189. assert len(names) == 23
  190. assert "1.0-3.0_MLCC" in names
  191. def test_lineage_hashes_are_recorded(processed):
  192. lineage = processed["result"].lineage
  193. for key in ["input_sha256", "params_hash", "output_hash", "engine"]:
  194. assert lineage.get(key)
  195. assert lineage["output_hash"].startswith("sha256:")
  196. def test_unparsed_records_are_flagged_for_review(processed):
  197. import csv
  198. with (processed["output"] / "未识别及提取异常.csv").open(encoding="utf-8-sig") as handle:
  199. rows = list(csv.DictReader(handle))
  200. assert rows, "应该存在需要人工复核的记录"
  201. assert any(row["flags"] for row in rows)