Pārlūkot izejas kodu

feat: 完善海关格式解析、精确聚合与遮掩数据演示

ly 3 dienas atpakaļ
vecāks
revīzija
411858e845
32 mainītis faili ar 2323 papildinājumiem un 415 dzēšanām
  1. 2 1
      agent.cordis.yml
  2. 14 0
      bff/dist/src/store/ledger.js
  3. 0 0
      bff/dist/src/store/ledger.js.map
  4. 28 0
      bff/dist/test/ledger.test.js
  5. 1 0
      bff/dist/test/ledger.test.js.map
  6. 17 0
      bff/src/store/ledger.ts
  7. 25 0
      bff/test/ledger.test.ts
  8. 48 0
      docs/06-工具增强与演示交付.md
  9. 81 0
      scripts/qualify_tool.py
  10. 43 0
      scripts/seed_demo.py
  11. 3 1
      scripts/start.mjs
  12. 191 0
      tools-service/tests/test_format_matrix.py
  13. 160 0
      tools-service/tests/test_legacy_fuzzy_brand.py
  14. 253 0
      tools-service/tests/test_legacy_irregular_model.py
  15. 74 0
      tools-service/tests/test_legacy_real_formats.py
  16. 2 1
      tools-service/tests/test_objects.py
  17. 1 1
      tools-service/tests/test_rules.py
  18. 3 3
      tools-service/toolsvc/customs/contract.py
  19. 104 0
      tools-service/toolsvc/customs/engine/brandmatch.py
  20. 14 2
      tools-service/toolsvc/customs/engine/numbers.py
  21. 7 1
      tools-service/toolsvc/customs/engine/reports.py
  22. 790 309
      tools-service/toolsvc/customs/engine/rules.py
  23. 50 53
      tools-service/toolsvc/customs/engine/storage.py
  24. 278 0
      tools-service/toolsvc/customs/engine/structure.py
  25. 3 3
      tools-service/toolsvc/customs/pipeline.py
  26. 8 32
      tools-service/toolsvc/objects.py
  27. 35 5
      tools-service/toolsvc/readers.py
  28. 85 0
      tools-service/toolsvc/source.py
  29. 0 0
      web/dist/assets/index-BqfzmH11.js
  30. 1 1
      web/dist/index.html
  31. 1 1
      web/src/pages/FilesPage.tsx
  32. 1 1
      web/src/pages/ProcessPage.tsx

+ 2 - 1
agent.cordis.yml

@@ -47,7 +47,7 @@ verification:
   # 核验失败时最多重试几次(架构文档规定 1 次)
   # 核验失败时最多重试几次(架构文档规定 1 次)
   maxRetries: 1
   maxRetries: 1
   # 允许模型明确标注"估算"的数字通过(标记为 flag)
   # 允许模型明确标注"估算"的数字通过(标记为 flag)
-  allowEstimateFlag: true
+  allowEstimateFlag: false
 
 
 systemPrompt: |
 systemPrompt: |
   你是中华人民共和国海关的电子元器件进出口数据分析助手,服务对象是海关业务人员。
   你是中华人民共和国海关的电子元器件进出口数据分析助手,服务对象是海关业务人员。
@@ -63,6 +63,7 @@ systemPrompt: |
   - 严禁编造、估算、心算任何数字。答案里出现的每一个金额、数量、百分比、指数,都必须原样来自本轮或之前轮次的工具返回值。
   - 严禁编造、估算、心算任何数字。答案里出现的每一个金额、数量、百分比、指数,都必须原样来自本轮或之前轮次的工具返回值。
   - 严禁自己推算"约等于"或"大概是"。需要换算时必须调用 `calculator`,并把输入值的 `from_call_id` / `row_hash` 一并传进去,让核验层可以追溯。
   - 严禁自己推算"约等于"或"大概是"。需要换算时必须调用 `calculator`,并把输入值的 `from_call_id` / `row_hash` 一并传进去,让核验层可以追溯。
   - 严禁把多张不同口径的表直接相加。`金额占比`/`数量占比` 的分母写在 `占比分母范围` 列,跨层级不可求和。
   - 严禁把多张不同口径的表直接相加。`金额占比`/`数量占比` 的分母写在 `占比分母范围` 列,跨层级不可求和。
+  - 对标有“已遮掩”的数据,金额或数量可能是区间字符串:仅能原样说明该区间,不可取中点、补出精确值或混入合成演示数据。仅回答用户所问指标,区间保留原始单位,不附加换算、四舍五入的占比或无关数字。原始申报明细未提供时须如实说明。
   - 严禁用"其他月份"或"0"填补缺失值。表里为空就是空,你要如实说明"该月无记录/指数不可计算"。
   - 严禁用"其他月份"或"0"填补缺失值。表里为空就是空,你要如实说明"该月无记录/指数不可计算"。
   - 只读数据,不修改数据。你没有写文件、执行命令的能力,也不要假装有。
   - 只读数据,不修改数据。你没有写文件、执行命令的能力,也不要假装有。
 
 

+ 14 - 0
bff/dist/src/store/ledger.js

@@ -26,6 +26,16 @@ function toNumber(value) {
     }
     }
     return null;
     return null;
 }
 }
+/** A masked interval is a literal observation, never an exact amount or midpoint. */
+function rangeBounds(field, value) {
+    if (typeof value !== 'string' || !/金额|数量|单价|价格/.test(field))
+        return [];
+    const match = value.trim().match(/^([+-]?\d+(?:\.\d+)?)\s*[-~~]\s*([+-]?\d+(?:\.\d+)?)$/);
+    if (!match)
+        return [];
+    const bounds = [Number(match[1]), Number(match[2])];
+    return bounds.every(Number.isFinite) && bounds[0] <= bounds[1] ? bounds : [];
+}
 export class SessionLedger {
 export class SessionLedger {
     db;
     db;
     now;
     now;
@@ -46,6 +56,10 @@ export class SessionLedger {
                         continue;
                         continue;
                     this.db.run(`INSERT INTO ledger_values(call_id,session_id,object_id,table_name,row_index,field,value_text,value_num)
                     this.db.run(`INSERT INTO ledger_values(call_id,session_id,object_id,table_name,row_index,field,value_text,value_num)
              VALUES(?,?,?,?,?,?,?,?)`, input.callId, input.sessionId, row._trace.object_id ?? '', row._trace.table ?? '', row._trace.row_index ?? -1, field, String(value), toNumber(value));
              VALUES(?,?,?,?,?,?,?,?)`, input.callId, input.sessionId, row._trace.object_id ?? '', row._trace.table ?? '', row._trace.row_index ?? -1, field, String(value), toNumber(value));
+                    for (const bound of rangeBounds(field, value)) {
+                        this.db.run(`INSERT INTO ledger_values(call_id,session_id,object_id,table_name,row_index,field,value_text,value_num)
+               VALUES(?,?,?,?,?,?,?,?)`, input.callId, input.sessionId, row._trace.object_id ?? '', row._trace.table ?? '', row._trace.row_index ?? -1, field, String(value), bound);
+                    }
                 }
                 }
             }
             }
             for (const [field, value] of Object.entries(input.extraValues ?? {})) {
             for (const [field, value] of Object.entries(input.extraValues ?? {})) {

Failā izmaiņas netiks attēlotas, jo tās ir par lielu
+ 0 - 0
bff/dist/src/store/ledger.js.map


+ 28 - 0
bff/dist/test/ledger.test.js

@@ -0,0 +1,28 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { Db } from '../src/store/db.js';
+import { SessionLedger } from '../src/store/ledger.js';
+import { verifyAnswer } from '../src/verify/verify.js';
+test('遮掩区间上下界溯源到原始单元格,不接受中点或任意区间数字', () => {
+    const db = new Db(':memory:');
+    try {
+        const ledger = new SessionLedger(db, () => '2026-11-17');
+        ledger.record({ callId: 'call_range', sessionId: 's1', tool: 'data_read', args: {}, objectIds: ['obj_masked'], rows: [
+                { values: { '实际金额(美元)': '890000000-900000000', '型号': '12345-67890' },
+                    _trace: { object_id: 'obj_masked', table: 'raw', row_index: 0, row_hash: 'sha256:test' } },
+            ] });
+        const result = verifyAnswer('进口金额为 890000000 至 900000000 美元(已遮掩区间)。', { ledger, sessionId: 's1', allowEstimateFlag: false });
+        assert.equal(result.verdict, 'accept');
+        assert.equal(result.verified, 2);
+        assert.equal(result.chips[0].object_id, 'obj_masked');
+        assert.equal(result.claims[0].matches[0].value_text, '890000000-900000000');
+        assert.equal(ledger.findValue('895000000', 's1').length, 0);
+        assert.equal(ledger.findValue('12345', 's1').length, 0);
+        assert.equal(ledger.findValue('890000000', 'other-session').length, 0);
+        assert.equal(verifyAnswer('进口额约 895000000 美元', { ledger, sessionId: 's1', allowEstimateFlag: false }).verdict, 'reject');
+    }
+    finally {
+        db.close();
+    }
+});
+//# sourceMappingURL=ledger.test.js.map

+ 1 - 0
bff/dist/test/ledger.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ledger.test.js","sourceRoot":"","sources":["../../test/ledger.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,EAAE,EAAE,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvD,IAAI,CAAC,8BAA8B,EAAE,GAAG,EAAE;IACxC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,EAAC,MAAM,EAAC,YAAY,EAAE,SAAS,EAAC,IAAI,EAAE,IAAI,EAAC,WAAW,EAAE,IAAI,EAAC,EAAE,EAAE,SAAS,EAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAC;gBAC5G,EAAE,MAAM,EAAC,EAAC,UAAU,EAAC,qBAAqB,EAAE,IAAI,EAAC,aAAa,EAAC;oBAC7D,MAAM,EAAC,EAAC,SAAS,EAAC,YAAY,EAAE,KAAK,EAAC,KAAK,EAAE,SAAS,EAAC,CAAC,EAAE,QAAQ,EAAC,aAAa,EAAC,EAAC;aACrF,EAAC,CAAC,CAAC;QACJ,MAAM,MAAM,GAAC,YAAY,CAAC,wCAAwC,EAAE,EAAC,MAAM,EAAE,SAAS,EAAC,IAAI,EAAE,iBAAiB,EAAC,KAAK,EAAC,CAAC,CAAC;QACvH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,EAAC,IAAI,CAAC,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAC,IAAI,CAAC,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,EAAC,eAAe,CAAC,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC;QACrE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,mBAAmB,EAAE,EAAC,MAAM,EAAC,SAAS,EAAC,IAAI,EAAC,iBAAiB,EAAC,KAAK,EAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACrH,CAAC;YAAS,CAAC;QAAA,EAAE,CAAC,KAAK,EAAE,CAAC;IAAA,CAAC;AACzB,CAAC,CAAC,CAAC"}

+ 17 - 0
bff/src/store/ledger.ts

@@ -55,6 +55,15 @@ function toNumber(value: unknown): number | null {
   return null;
   return null;
 }
 }
 
 
+/** A masked interval is a literal observation, never an exact amount or midpoint. */
+function rangeBounds(field: string, value: unknown): number[] {
+  if (typeof value !== 'string' || !/金额|数量|单价|价格/.test(field)) return [];
+  const match = value.trim().match(/^([+-]?\d+(?:\.\d+)?)\s*[-~~]\s*([+-]?\d+(?:\.\d+)?)$/);
+  if (!match) return [];
+  const bounds = [Number(match[1]), Number(match[2])];
+  return bounds.every(Number.isFinite) && bounds[0] <= bounds[1] ? bounds : [];
+}
+
 export class SessionLedger {
 export class SessionLedger {
   constructor(
   constructor(
     private readonly db: Db,
     private readonly db: Db,
@@ -100,6 +109,14 @@ export class SessionLedger {
             String(value),
             String(value),
             toNumber(value),
             toNumber(value),
           );
           );
+          for (const bound of rangeBounds(field, value)) {
+            this.db.run(
+              `INSERT INTO ledger_values(call_id,session_id,object_id,table_name,row_index,field,value_text,value_num)
+               VALUES(?,?,?,?,?,?,?,?)`,
+              input.callId, input.sessionId, row._trace.object_id ?? '', row._trace.table ?? '',
+              row._trace.row_index ?? -1, field, String(value), bound,
+            );
+          }
         }
         }
       }
       }
       for (const [field, value] of Object.entries(input.extraValues ?? {})) {
       for (const [field, value] of Object.entries(input.extraValues ?? {})) {

+ 25 - 0
bff/test/ledger.test.ts

@@ -0,0 +1,25 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { Db } from '../src/store/db.js';
+import { SessionLedger } from '../src/store/ledger.js';
+import { verifyAnswer } from '../src/verify/verify.js';
+
+test('遮掩区间上下界溯源到原始单元格,不接受中点或任意区间数字', () => {
+  const db = new Db(':memory:');
+  try {
+    const ledger = new SessionLedger(db, () => '2026-11-17');
+    ledger.record({callId:'call_range', sessionId:'s1', tool:'data_read', args:{}, objectIds:['obj_masked'], rows:[
+      { values:{'实际金额(美元)':'890000000-900000000', '型号':'12345-67890'},
+        _trace:{object_id:'obj_masked', table:'raw', row_index:0, row_hash:'sha256:test'}},
+    ]});
+    const result=verifyAnswer('进口金额为 890000000 至 900000000 美元(已遮掩区间)。', {ledger, sessionId:'s1', allowEstimateFlag:false});
+    assert.equal(result.verdict, 'accept');
+    assert.equal(result.verified, 2);
+    assert.equal(result.chips[0].object_id, 'obj_masked');
+    assert.equal(result.claims[0].matches[0].value_text, '890000000-900000000');
+    assert.equal(ledger.findValue('895000000','s1').length,0);
+    assert.equal(ledger.findValue('12345','s1').length,0);
+    assert.equal(ledger.findValue('890000000','other-session').length,0);
+    assert.equal(verifyAnswer('进口额约 895000000 美元', {ledger,sessionId:'s1',allowEstimateFlag:false}).verdict, 'reject');
+  } finally {db.close();}
+});

+ 48 - 0
docs/06-工具增强与演示交付.md

@@ -0,0 +1,48 @@
+# 海关统计工具增强与 Demo 交付(2026-11-17)
+
+已在用户指定的本地项目修改代码并接通 DeepSeek。密钥只在被 Git 忽略的本地 `.env`,权限为 600,不写入报告或源码。运行数据、密钥、虚拟环境及测试临时产物均不随源码提交。
+
+## 启动与演示
+
+在项目根目录执行 `node scripts/start.mjs`,浏览器访问 http://127.0.0.1:8787 。启动脚本会自动优先使用项目 `.venv`;按 Ctrl+C 停止,或执行 `node scripts/stop.mjs`。
+
+预置两个工作区:
+
+- **合成样本演示(可复现)**:原始样本及 23 张统计结果,可演示处理、精确问答、来源核验、行级下钻。先问“2025年1月 MLCC 的进口额和数量是多少?”,再问“这个月的金额指数是多少?”;也可在数据处理页选择原始样本重新生成结果。
+- **真实统计参考(已遮掩)**:用户提供的 16 张统计 CSV,保留原始区间文本。可以问“2025年1月 MLCC 的进口金额和数量各在哪个区间?”或查询已保留的指数。每个文件只有一张 `raw` 表,表示导入文件的原文表。它们是已完成的统计结果,不要再次作为原始申报数据处理。
+
+遮掩结果包只有统计表,没有原始申报明细,不能据此复现当时的识别率、确认约 35% 未识别记录的构成,或证明真实全量统计与旧版完全一致。区间不会换成中点或精确金额;跨层级重复统计也不能直接相加。
+
+## 改动
+
+1. 接入用户提供的 `component_trade_promax(1).zip` 中经过回归测试的 `rules/structure/brandmatch` 逻辑,同时保留现有处理器契约。前端与 Agent 共用它;没有引入第二条统计链路。
+2. 覆盖括号中的中英品牌、品牌/型号标签变体、标签和值分开、空格/下划线/斜杠分段、型号里的斜杠、多品牌冲突、同品牌重复、明确无品牌/无型号、型号分类及应用二轮识别。近似、包含或推断结果保留方法、候选、分数和异常标记。80 分只是拼写匹配门槛,不是准确概率;包含匹配是旧版独立的兜底规则,可能低于 80 分。
+3. 上传和处理统一流式读取,支持 UTF-8/BOM、GB18030、带 BOM 的 UTF-16,逗号/Tab/分号/竖线,CSV/TSV/TXT 和首张 XLSX/XLSM 工作表。完整流验证编码,避免长 ASCII 前缀误判;单字段上限 10MiB;跨行字段保留正确物理结束行号。
+4. 支持 MONORSUM_USD、旧 MONORSUM 和中文表头、BOM、全角编码及 Excel 整数 `.0` 导出;按列名解析,不依赖列序。重复别名、缺列、不配对引号明确报错,列数错行记入排除明细,不猜补。
+5. 修复年份范围未实际限制 fact 的问题;按 ISO 周四归属年判断。空 G_MODEL 沿用交付旧脚本口径排除。范围内非空但品牌/型号识别不全的记录,金额数量仍计入总量,冲突记录不强行分配品牌、不重复计数。
+6. 美元列可清理合法千分位、全角数字、USD/$/美元、会计负数。非法分组、非有限值、人民币数值、遮掩区间保持未知,严格求和向上传播缺数。只有人民币金额列时明确拒绝,避免把人民币误算美元;同时存在明确美元列时使用美元列。
+7. 自定义统计以及 Agent `data_read` 的 sum/avg/min/max 均从原始文本使用 Decimal,避免浮点大整数丢位、0.1+0.2 尾差、缺失成员静默跳过。REAL 影子列仍只用于筛选/排序;数值特别接近时排序仍受浮点精度限制。处理线程继承 50 位精度。
+8. 新增识别计数与排除原因,写入 run_summary、结果对象元数据和“数据质量摘要”。原始行、解析结果、异常保留可复核文件。新增读取器和解析器有界缓存、型号前缀/片段索引;不逐条调用大模型。
+
+9. 遮掩区间上下界记录到会话台账,关联原始单元格和行哈希;不将中点当作观测值。关闭无来源估算数字放行,核验不通过时要求模型修正。
+
+## 验证与可复现命令
+
+- Python:194 项通过(原测试加旧版格式回归、生成式格式矩阵、金额/缺数/范围/精度测试)。命令:`.venv/bin/python -m pytest -c tools-service/pyproject.toml tools-service/tests`。
+- BFF:59 项通过;前端:27 项通过;前端/BFF 构建成功。
+- 浏览器实测:在真实遮掩工作区查询金额/数量区间,四个上下界及记录数均溯源,核验 accept;点击数字可查看原始区间、行号和行哈希。
+- 真实服务 + DeepSeek:33 项端到端检查通过,包含上传、处理 23 张表、下载、溯源、台账和核验通过后的答案交付。命令:`CDA_PYTHON="$PWD/.venv/bin/python" node scripts/e2e.mjs --keep`。
+- 旧脚本对比:固定 16 种写法的品牌、型号、国别、类目、应用和分类标志均与用户提供脚本一致;品牌与型号均提取出的测试用例由增强前 4/16 增至 11/16。这是有意选择的格式测试集合,不能外推为真实数据识别率。
+- 100,000 条合成记录(16 种重复申报)执行流式读取、解析、审计入库、月度汇总和索引,共 5.850 秒,全部纳入,精确金额合计 10000.00 美元。该计时不含上传索引、规则加载、完整报表、模型请求,也不是百万条不同申报的生产性能承诺。
+
+对比脚本:`.venv/bin/python scripts/qualify_tool.py --legacy /path/to/component_trade_promax --before .run/qualification/before_rules.py --rows 100000`。
+
+新增工作区可用 `CDA_DATA_DIR="$PWD/data" .venv/bin/python scripts/seed_demo.py --masked-dir /path/to/unpacked/csvs` 重建。相同名称工作区已存在时跳过,避免重复导入。
+
+## 文件和后续边界
+
+增强前文件备份位于 `.run/backup-before-tool-upgrade/`,修改清单位于 `.run/upgrade-manifest.json`。测试/对比证据位于 `.run/qualification/`。
+
+目前是本机单机 demo:长任务仍由服务进程中的后台线程执行,没有断点续跑或持久化任务队列;重启可能中断任务。XLSX 只读第一张表,旧 `.xls`、混合编码、无分隔定宽文本、自动换汇不在支持范围。真实全量的高唯一申报率、高基数结果维度和报表时间仍需取得原始数据后单独压测。
+
+测试仅有一项依赖弃用提醒:新版本 Starlette 推荐 httpx2;本轮所有 FastAPI TestClient 接口测试通过。

+ 81 - 0
scripts/qualify_tool.py

@@ -0,0 +1,81 @@
+"""Reproducible comparison with the supplied legacy parser and ingestion benchmark.
+
+Usage: .venv/bin/python scripts/qualify_tool.py --legacy ../reference_20261117/component_trade_promax --rows 100000
+All generated data is synthetic. Timings do not estimate production throughput.
+"""
+import argparse
+import csv
+import importlib.util
+import json
+import sys
+import time
+from pathlib import Path
+from decimal import Decimal
+
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'tools-service'))
+from toolsvc.customs.engine.rules import Rules
+from toolsvc.customs.engine.storage import ingest, Store
+
+
+def main():
+    parser=argparse.ArgumentParser()
+    parser.add_argument('--legacy',type=Path,required=True)
+    parser.add_argument('--rows',type=int,default=100000)
+    parser.add_argument('--before',type=Path)
+    args=parser.parse_args()
+    out=ROOT/'.run/qualification'; out.mkdir(parents=True,exist_ok=True)
+    folder=ROOT/'tools-service/toolsvc/customs/rules'
+    mapping=folder/'数据映射规则.xlsx'; fields=folder/'进出口数据字段需求20260819.xlsx'
+    sys.path.insert(0,str(args.legacy.resolve()))
+    from trade0909.rules import Rules as LegacyRules
+    legacy=LegacyRules(mapping, fields); current=Rules(mapping, fields)
+    # Independent baseline: agent implementation before this change.
+    original=args.before
+    before=None
+    if original and original.exists():
+        spec=importlib.util.spec_from_file_location('before_rules',original)
+        mod=importlib.util.module_from_spec(spec); spec.loader.exec_module(mod)
+        before=mod.Rules(mapping, fields)
+    texts=[
+        '4|0|多层|陶瓷|MURATA|TEST-1|||',
+        '品牌:MURATA;型号:TEST-1', '品牌名称:村田;型号规格:TEST-1',
+        '村田(MURATA)|TEST-1', 'MURATA/TEST-1/计算机用',
+        'MURATA_TEST-1', '0 0 多层片式 瓷介 MURATA TEST-1',
+        '品牌为MURATA,型号为TEST-1', '(品牌)MURATA(型号)TEST-1',
+        '品牌|MURATA|型号|TEST-1', '品牌:MURTA;型号:TEST-1',
+        '品牌:MURATA,SAMSUNG;型号:TEST-1',
+        '品牌:ABACUS;型号:A5E30295495R2W', '品牌:无品牌;型号:无型号',
+        '品牌:MURATA;型号:1/10W', '品牌:MURATA;用途:5G基站',
+    ]
+    records=[]
+    keys=['brand','model','brand_country','c1','c2','c3','c4','app','mapped']
+    for text in texts:
+        old=legacy.parse('8532241000',text); new=current.parse('8532241000',text)
+        prev=before.parse('8532241000',text) if before else None
+        records.append(dict(raw=text,legacy={k:old[k] for k in keys},current={k:new[k] for k in keys},
+            legacy_equal=all(old[k]==new[k] for k in keys),
+            before_pair=[prev['brand'],prev['model']] if prev else None))
+    source=out/'synthetic.csv'
+    with source.open('w',encoding='utf-8-sig',newline='') as f:
+        w=csv.writer(f);w.writerow(['I_E_FLAG','HS_CODE','YEAR','WEEK','MONORSUM','COUNT','G_MODEL'])
+        for i in range(args.rows):
+            w.writerow(['I','8532241000','2025',str(i%52+1),'0.10','1',texts[i%len(texts)]])
+    bench=out/f'bench-{time.time_ns()}';bench.mkdir()
+    start=time.perf_counter()
+    db,stats=ingest(dict(input_csv=str(source),directions=['I'],start_year=2025,end_year=2026,cutoff_date='2026-12-31'),current,bench)
+    elapsed=time.perf_counter()-start
+    totals=Store(db).total({});db.close()
+    assert stats['included']==args.rows
+    assert totals['amount']==Decimal('0.1')*args.rows
+    report=dict(comparison=records, equal_to_legacy=sum(r['legacy_equal'] for r in records),
+        cases=len(records),before_recognized_pairs=sum(bool(r['before_pair'] and all(r['before_pair'])) for r in records),
+        current_recognized_pairs=sum(bool(r['current']['brand'] and r['current']['model']) for r in records),
+        synthetic_benchmark=dict(rows=args.rows,distinct_declarations=len(texts),seconds=round(elapsed,3),
+            rows_per_second=round(args.rows/elapsed),amount=str(totals['amount']),included=stats['included'],
+            scope='stream reading, parsing, audited SQLite ingestion, monthly aggregation and indexes; excludes full reports and uploads'))
+    (out/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2))
+    print(json.dumps({k:v for k,v in report.items() if k!='comparison'},ensure_ascii=False,indent=2))
+    assert report['equal_to_legacy']==len(records)
+
+if __name__=='__main__':main()

+ 43 - 0
scripts/seed_demo.py

@@ -0,0 +1,43 @@
+"""Create a synthetic demo workspace; optionally import masked CSV references.
+
+Run with CDA_DATA_DIR set to the service's absolute data directory.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'tools-service'))
+from toolsvc.db import database
+from toolsvc.objects import store
+from toolsvc.runs import runs
+
+
+def main():
+    parser=argparse.ArgumentParser()
+    parser.add_argument('--masked-dir',type=Path)
+    args=parser.parse_args()
+    existing={w['name']:w for w in store.list_workspaces()}
+    name='合成样本演示(可复现)'
+    if name in existing:
+        workspace=existing[name]
+    else:
+        workspace=store.create_workspace(name,'使用合成样本演示原始数据处理、统计问答和行级溯源;不代表真实贸易数据。')
+        source=store.register_upload(workspace['id'],'合成演示_small.csv',(ROOT/'samples/small.csv').read_bytes())
+        run=runs.create(workspace['id'],source['id'],options={'demo':True})
+        import time
+        while run['status'] in {'queued','running'}:
+            time.sleep(.3);run=runs.get(run['id'])
+        if run['status']!='succeeded':raise RuntimeError(run.get('error'))
+    imported=[]
+    if args.masked_dir:
+        name='真实统计参考(已遮掩)'
+        if name not in existing:
+            masked=store.create_workspace(name,'2026年11月11日结果:16张已遮掩统计表。金额数量中的区间不是精确值,不可取中点或对重复层级求和。此处没有原始申报明细。')
+            for path in sorted(args.masked_dir.glob('*.csv')):
+                item=store.register_upload(masked['id'],'【已遮掩】'+path.name,path.read_bytes(),kind='stat',producer='import_masked_reference')
+                imported.append({'name':item['name'],'rows':item['rows']})
+    print(json.dumps({'synthetic_workspace':workspace['id'],'masked_imports':imported},ensure_ascii=False,indent=2))
+
+if __name__=='__main__': main()

+ 3 - 1
scripts/start.mjs

@@ -21,7 +21,8 @@ const PID_FILE = path.join(RUN_DIR, 'pids.json');
 const args = new Set(process.argv.slice(2));
 const args = new Set(process.argv.slice(2));
 
 
 const IS_WINDOWS = process.platform === 'win32';
 const IS_WINDOWS = process.platform === 'win32';
-const PYTHON = process.env.CDA_PYTHON ?? (IS_WINDOWS ? 'python' : 'python3');
+const VENV_PYTHON = path.join(ROOT, '.venv', IS_WINDOWS ? 'Scripts/python.exe' : 'bin/python');
+let PYTHON = process.env.CDA_PYTHON ?? (existsSync(VENV_PYTHON) ? VENV_PYTHON : (IS_WINDOWS ? 'python' : 'python3'));
 
 
 // --------------------------------------------------------------------- 输出
 // --------------------------------------------------------------------- 输出
 const C = {
 const C = {
@@ -173,6 +174,7 @@ async function main() {
 
 
   const total = 7;
   const total = 7;
   loadDotEnv();
   loadDotEnv();
+  PYTHON = process.env.CDA_PYTHON ?? PYTHON;
   mkdirSync(RUN_DIR, { recursive: true });
   mkdirSync(RUN_DIR, { recursive: true });
 
 
   // 1. Node
   // 1. Node

+ 191 - 0
tools-service/tests/test_format_matrix.py

@@ -0,0 +1,191 @@
+"""Generated format matrix, accounting invariants and precision regressions."""
+import csv
+import sqlite3
+from decimal import Decimal
+from pathlib import Path
+
+import pytest
+
+from toolsvc.customs import CustomsOptions, CustomsProcessRequest, load_processor
+from toolsvc.customs.engine.numbers import number
+from toolsvc.customs.engine.rules import Rules
+from toolsvc.customs.engine.storage import Store, ingest
+from toolsvc.objects import _iter_rows, store
+from toolsvc.source import iter_source
+
+HEAD = ['I_E_FLAG', 'HS_CODE', 'YEAR', 'WEEK', 'MONORSUM_USD', 'COUNT', 'G_MODEL']
+ROW = ['I', '8532241000', '2025', '1', '1234.50', '10', '品牌:MURATA;型号:TEST-1']
+RULE_DIR = Path(__file__).resolve().parents[1] / 'toolsvc/customs/rules'
+
+
+@pytest.fixture(scope='module')
+def rules():
+    return Rules(RULE_DIR / '数据映射规则.xlsx', RULE_DIR / '进出口数据字段需求20260819.xlsx')
+
+
+def config(path, **extra):
+    return dict(input_csv=str(path), start_year=2025, end_year=2026,
+                directions=['I'], cutoff_date='2026-12-31', **extra)
+
+
+@pytest.mark.parametrize('encoding', ['utf-8', 'utf-8-sig', 'gb18030', 'utf-16'])
+@pytest.mark.parametrize('delimiter', [',', '\t', ';', '|'])
+def test_upload_and_ingest_same_formats(tmp_path, rules, encoding, delimiter):
+    path = tmp_path / 'input.csv'
+    with path.open('w', encoding=encoding, newline='') as f:
+        w = csv.writer(f, delimiter=delimiter)
+        w.writerow(HEAD)
+        w.writerow(ROW)
+    headers, iterator = _iter_rows(path)
+    assert headers == HEAD
+    assert list(iterator) == [ROW]
+    db, stats = ingest(config(path), rules, tmp_path)
+    try:
+        assert stats['included'] == 1
+        assert Store(db).total({})['amount'] == Decimal('1234.50')
+    finally:
+        db.close()
+
+
+@pytest.mark.parametrize('raw,expected', [
+    ('1,234.50', '1234.50'), ('1,234.50', '1234.50'),
+    ('1 234.50', '1234.50'), ('$1,234.50', '1234.50'),
+    ('1234.50美元', '1234.50'), ('(1,234.50)', '-1234.50'),
+    ('−100', '-100'), ('1.2345e3', '1234.5'),
+    ('12,34', None), ('1,2,3', None), ('1.234,50', None),
+    ('123元', None), ('¥123', None), ('123RMB', None),
+    ('NaN', None), ('Infinity', None), ('890000000-900000000', None),
+])
+def test_numbers_and_masked_ranges(raw, expected):
+    assert number(raw) == (Decimal(expected) if expected is not None else None)
+
+
+def test_chinese_headers_and_excel_numeric_codes(tmp_path, rules):
+    from openpyxl import Workbook
+    p = tmp_path / 'input.xlsx'
+    wb = Workbook()
+    ws = wb.active
+    ws.append(['进出口标志', '商品编码', '年度', 'ISO周', '金额(美元)', '成交数量', '申报要素'])
+    ws.append(['进口', 8532241000, 2025, 1, 100.5, 10, ROW[-1]])
+    wb.save(p)
+    db, stats = ingest(config(p), rules, tmp_path)
+    try:
+        assert stats['included'] == 1
+        assert Store(db).total({})['amount'] == Decimal('100.5')
+    finally:
+        db.close()
+
+
+def test_all_in_scope_unknowns_are_counted_once(tmp_path, rules):
+    p = tmp_path / 'in.csv'
+    texts = ['品牌:ABACUS;型号:A5E30295495R2W', '品牌:无品牌;型号:无型号',
+             '品牌:MURATA、SAMSUNG;型号:TEST-1', ROW[-1]]
+    with p.open('w', encoding='utf-8', newline='') as f:
+        w = csv.writer(f); w.writerow(HEAD)
+        for text in texts: w.writerow(ROW[:4]+['100', '10', text])
+    db, stats = ingest(config(p), rules, tmp_path)
+    try:
+        assert stats['included'] == 4
+        assert Store(db).total({})['amount'] == Decimal('400')
+        assert sum(r['n'] for r in Store(db).aggregate(('brand',))) == 4
+        assert stats['recognition']['brand_conflict'] >= 1
+        assert stats['recognition']['explicit_no_brand'] == 1
+    finally:
+        db.close()
+
+
+def test_multiline_physical_line_numbers(tmp_path):
+    p = tmp_path / 'multiline.csv'
+    with p.open('w', encoding='utf-8', newline='') as f:
+        w = csv.writer(f); w.writerow(HEAD)
+        w.writerow(ROW[:-1]+['品牌:MURATA\n型号:TEST-1']); w.writerow(ROW)
+    rows = list(iter_source(p))
+    assert [(r[0], r[1]) for r in rows] == [(0, 1), (1, 3), (2, 4)]
+
+
+def test_year_range_applies_to_fact(tmp_path, rules):
+    p = tmp_path / 'old.csv'
+    with p.open('w', newline='') as f:
+        w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:2]+['2024']+ROW[3:])
+    db, stats = ingest(config(p), rules, tmp_path)
+    try:
+        assert stats['included'] == 0 and stats['excluded'] == 1
+    finally:
+        db.close()
+
+
+def test_duplicate_alias_and_currency_fail_clearly(tmp_path, rules):
+    for name, headers, match in [('duplicate', HEAD+['金额'], '重复'),
+                                  ('rmb', HEAD[:4]+['金额:人民币']+HEAD[5:], '人民币')]:
+        out=tmp_path/name; out.mkdir(); p=out/'in.csv'
+        p.write_text(','.join(headers)+'\n', encoding='utf-8')
+        with pytest.raises(ValueError, match=match): ingest(config(p), rules, out)
+
+
+def test_bad_quotes_fail_without_silent_repair(tmp_path):
+    p=tmp_path/'bad.csv'; p.write_text(','.join(HEAD)+'\nI,8532241000,2025,1,100,10,"未闭合\n')
+    with pytest.raises(ValueError, match='CSV 第'): list(iter_source(p))
+
+
+def test_long_cell_and_bom_boundary(tmp_path):
+    p=tmp_path/'long.csv'
+    text='中文'*70000
+    with p.open('w', encoding='utf-8', newline='') as f:
+        w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+[text])
+    assert list(iter_source(p))[1][3][-1] == text
+
+
+@pytest.mark.parametrize('amounts,expected', [(['0.1','0.2'],'0.3'),
+    (['9007199254740993','1'],'9007199254740994'), (['1','bad'], None)])
+def test_custom_aggregate_precision_and_unknowns(tmp_path, amounts, expected):
+    p=tmp_path/'input.csv'
+    with p.open('w', newline='') as f:
+        w=csv.writer(f); w.writerow(HEAD)
+        for amount in amounts: w.writerow(ROW[:4]+[amount]+ROW[5:])
+    result=load_processor().process(CustomsProcessRequest(str(p), str(tmp_path/'out'),
+        operation='custom_aggregate', options=CustomsOptions(group_by=['brand'])))
+    with sqlite3.connect(result.sqlite_path) as db:
+        actual=db.execute('SELECT "实际金额(美元)" FROM s_自定义聚合').fetchone()[0]
+    assert actual == (expected if expected is not None else '')
+
+
+@pytest.mark.parametrize('fn,expected', [('sum', '9007199254740994'), ('avg', '4503599627370497'), ('min', '1'), ('max', '9007199254740993')])
+def test_agent_data_read_aggregates_are_exact(fn, expected):
+    from toolsvc.readers import read_rows
+    workspace = store.create_workspace('精度测试')
+    obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n9007199254740993\n1\n')
+    result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':fn,'alias':'value'}]})
+    assert result['rows'][0]['values']['value'] == expected
+
+
+def test_agent_data_read_unknown_propagates():
+    from toolsvc.readers import read_rows
+    workspace = store.create_workspace('缺数测试')
+    obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n100\nmasked-range\n')
+    result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':'sum','alias':'value'}]})
+    assert result['rows'][0]['values']['value'] is None
+
+
+def test_processing_threads_keep_decimal_precision():
+    from concurrent.futures import ThreadPoolExecutor
+    from decimal import getcontext
+    with ThreadPoolExecutor(max_workers=1) as executor:
+        assert executor.submit(lambda:getcontext().prec).result() == 50
+
+
+def test_gb_encoding_after_long_ascii_prefix(tmp_path):
+    p=tmp_path/'late.csv'
+    with p.open('w', encoding='gb18030', newline='') as f:
+        w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+['A'*70000+'品牌:村田'])
+    assert list(iter_source(p))[1][3][-1].endswith('品牌:村田')
+
+
+def test_usd_amount_selected_with_rmb_extra_column(tmp_path, rules):
+    p=tmp_path/'both.csv'
+    with p.open('w', newline='') as f:
+        w=csv.writer(f);w.writerow(HEAD+['MONORSUM_RMB']);w.writerow(ROW+['8000'])
+    db,stats=ingest(config(p),rules,tmp_path)
+    try:
+        assert stats['included']==1
+        assert Store(db).total({})['amount']==Decimal('1234.50')
+    finally:db.close()

+ 160 - 0
tools-service/tests/test_legacy_fuzzy_brand.py

@@ -0,0 +1,160 @@
+"""80% boundary, candidate ranking, parsing and exclusion regressions."""
+import csv
+import sys
+import tempfile
+import unittest
+from decimal import Decimal
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1] / 'toolsvc' / 'customs'
+sys.path.insert(0, str(ROOT))
+from toolsvc.customs.engine.brandmatch import BrandMatcher
+from toolsvc.customs.engine.rules import Rules
+from toolsvc.customs.engine.storage import ingest, REQUIRED
+
+
+class MatcherTests(unittest.TestCase):
+    def matcher(self, aliases):
+        return BrandMatcher({k: {v} for k, v in aliases.items()}, {'无品牌', '无'})
+
+    def test_case_space_and_special_characters(self):
+        m = self.matcher({'MURATA': '村田'})
+        labels, _, method, score, _ = m.match(('m-u_r.a t a',))
+        self.assertEqual(labels, ('村田',))
+        self.assertEqual((method, score), ('标点归一', 100))
+        self.assertEqual(self.matcher({'TI': '德州仪器'}).match(('t.i.',))[0], ('德州仪器',))
+
+    def test_exact_eighty_percent_is_inclusive(self):
+        m = self.matcher({'ABCDE': 'A'})
+        self.assertEqual(m.match(('ABCDX',))[3], 80)
+        self.assertEqual(m.match(('ABCXX',))[0], ())
+
+    def test_choose_highest_similarity(self):
+        m = self.matcher({'XBCDE': '低', 'ABCDEF': '高'})
+        labels, _, _, score, candidates = m.match(('ABCDE',))
+        self.assertEqual(labels, ('高',))
+        self.assertAlmostEqual(score, 100 * 5 / 6)
+        self.assertEqual(len(candidates), 2)
+
+    def test_equal_top_scores_are_ambiguous(self):
+        m = self.matcher({'ABCDX': 'A', 'ABCDY': 'B'})
+        self.assertEqual(set(m.match(('ABCDE',))[0]), {'A', 'B'})
+
+    def test_twenty_percent_edits_on_long_names(self):
+        m = self.matcher({'ABCDEFGHIJ': 'A'})
+        self.assertEqual(m.match(('ABXDEFGHIY',))[3], 80)
+
+    def test_placeholders_and_short_fragments(self):
+        m = self.matcher({'无品牌': '错误', 'TI': '错误', 'MURATA': 'M'})
+        for raw in ('无品牌', 'T1', 'XYZ123', 'GRM188R71H104'):
+            self.assertEqual(m.match((raw,))[0], (), raw)
+
+    def test_substring_containment_below_eighty_percent(self):
+        # “日本兴亚”只包含别名“兴亚”,覆盖率 50%,旧逻辑直接放弃。
+        m = self.matcher({'兴亚': 'KOA'})
+        labels, _, method, score, _ = m.match(('日本兴亚',))
+        self.assertEqual(labels, ('KOA',))
+        self.assertEqual(method, '子串包含')
+        self.assertAlmostEqual(score, 50)
+        # 关掉包含匹配后恢复旧行为。
+        strict = BrandMatcher({'兴亚': {'KOA'}}, set(), allow_contains=False)
+        self.assertEqual(strict.match(('日本兴亚',))[0], ())
+
+    def test_containment_keeps_word_boundary_for_latin(self):
+        m = self.matcher({'ABCD': 'A', 'YAGEO': 'Y'})
+        # 被字母包住的片段不算命中(ABCD 在 XABCDY 里没有词边界)。
+        self.assertEqual(m.match(('XABCDY',))[0], ())
+        # 被中文或分隔符隔开的片段算命中。
+        self.assertEqual(m.match(('前标YAGEO后标',))[0], ('Y',))
+        # 拉丁名短于 4 个字符不做低覆盖率包含,避免缩写误命中。
+        self.assertEqual(self.matcher({'KOA': 'K'}).match(('XKOAX',))[0], ())
+
+
+class IntegrationTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx',
+                          ROOT / 'rules/进出口数据字段需求20260819.xlsx')
+
+    def test_fuzzy_in_real_field_formats(self):
+        for raw in ('品牌:MURTA;型号:TEST-1',
+                    'MURTA/TEST-1/计算机用',
+                    '4|3|片式|瓷介|MURTA|TEST-1'):
+            p = self.rules.parse('8532241000', raw)
+            self.assertEqual((p['brand'], p['model']), ('MURATA', 'TEST-1'), raw)
+            self.assertAlmostEqual(p['brand_match_score'], 83.333333)
+            self.assertEqual(p['brand_match_method'], '拼写近似')
+
+    def test_labelled_and_standalone_punctuated_brand(self):
+        for raw in ('品牌:m-u_r.a t a;型号:TEST-1', 'm-u_r.a t a'):
+            p = self.rules.parse('8532241000', raw)
+            self.assertEqual(p['brand'], 'MURATA', raw)
+            self.assertEqual(p['brand_match_score'], 100)
+
+    def test_model_or_usage_label_is_not_a_brand(self):
+        for raw in ('型号:MURTA', '用途:MURTA'):
+            self.assertEqual(self.rules.parse('8532241000', raw)['brand'], '')
+
+    def test_alias_fragment_recovers_brand_inside_longer_text(self):
+        # 海关写法“品牌:日本兴亚”,规则表里只有“KOA/兴亚”“KOA,日本兴亚”等整串。
+        for raw in ('品牌:日本兴亚', '日本兴亚', '品:KOA等 中文名称:兴亚等'):
+            p = self.rules.parse('8532241000', raw)
+            self.assertEqual(p['brand'], 'KOA', raw)
+        p = self.rules.parse('8532241000', '品牌:日本兴亚电子')
+        self.assertEqual(p['brand'], 'KOA')
+        self.assertEqual(p['brand_match_method'], '子串包含')
+        self.assertIn('品牌包含匹配', p['flags'])
+
+    def test_model_containment_recovers_embedded_model(self):
+        # 已知型号被包在更长的申报文本里,旧逻辑整条判为未识别。
+        for raw in ('型号:全新原装K4RAH086VE-E0000EL',
+                    '品牌:SAMSUNG;型号:贴片K4RAH086VE-E0000EL'):
+            p = self.rules.parse('8542321010', raw)
+            self.assertEqual(p['model'], 'K4RAH086VE-E0000EL', raw)
+            self.assertIn('型号包含匹配', p['flags'])
+            self.assertEqual(p['c2'], 'DRAM', raw)
+        # 括号包裹的值现在先被清理成干净型号、直接精确命中,不再需要包含匹配兜底
+        p = self.rules.parse('8542321010', '型号:(K4RAH086VE-E0000EL)')
+        self.assertEqual(p['model'], 'K4RAH086VE-E0000EL')
+        self.assertEqual(p['c2'], 'DRAM')
+
+    def test_model_containment_only_for_memory_hs(self):
+        # 型号表只覆盖存储器 4 个 HS,其他 HS 不能凭猜测套用。
+        p = self.rules.parse('8532241000', '型号:全新原装K4RAH086VE-E0000EL')
+        self.assertNotEqual(p['model'], 'K4RAH086VE-E0000EL')
+        self.assertEqual(p['mapped'], 0)
+
+    def test_containment_can_be_switched_off(self):
+        strict = Rules(ROOT / 'rules/数据映射规则.xlsx',
+                       ROOT / 'rules/进出口数据字段需求20260819.xlsx',
+                       fuzzy_contains=False)
+        self.assertEqual(strict.parse('8532241000', '品牌:日本兴亚电子')['brand'], '')
+        self.assertNotEqual(strict.parse('8542321010', '型号:全新原装K4RAH086VE-E0000EL')['model'],
+                            'K4RAH086VE-E0000EL')
+
+    def test_empty_gmodel_excluded_from_all_statistics(self):
+        with tempfile.TemporaryDirectory() as folder:
+            out = Path(folder)
+            source = out / 'input.csv'
+            with source.open('w', encoding='utf-8-sig', newline='') as f:
+                writer = csv.writer(f)
+                writer.writerow(REQUIRED)
+                for value in ('', ' \t\u3000', '品牌:MURTA;型号:TEST-1'):
+                    writer.writerow(['I', '8532241000', '2025', '1', '100', '10', value])
+            config = dict(input_csv=str(source), encoding='utf-8-sig', delimiter=',',
+                          cutoff_date='2026-08-31', directions=['I'], start_year=2025, end_year=2026)
+            db, stats = ingest(config, self.rules, out)
+            try:
+                self.assertEqual((stats['total'], stats['included'], stats['excluded']), (3, 1, 2))
+                self.assertEqual(db.execute('SELECT count(*) FROM audit').fetchone()[0], 1)
+                for table in ('fact', 'monthly'):
+                    rows = db.execute(f'SELECT amount, quantity FROM {table}').fetchall()
+                    self.assertEqual(sum(Decimal(r[0]) for r in rows), 100)
+                    self.assertEqual(sum(Decimal(r[1]) for r in rows), 10)
+                self.assertEqual(stats['exclusion_reasons']['G_MODEL为空'], 2)
+            finally:
+                db.close()
+
+
+if __name__ == '__main__':
+    unittest.main()

+ 253 - 0
tools-service/tests/test_legacy_irregular_model.py

@@ -0,0 +1,253 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from toolsvc.customs.engine.rules import Rules
+from toolsvc.customs.engine.structure import split_attributes
+
+
+ROOT = Path(__file__).resolve().parents[1] / 'toolsvc' / 'customs'
+
+
+class IrregularModelTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx',
+                          ROOT / 'rules/进出口数据字段需求20260819.xlsx')
+
+    def test_quoted_semicolon_and_sentence_usage(self):
+        parsed = self.rules.parse('8532241000',
+                                  '"品牌:MURATA;型号:TEST-1;用途:这句话说明用于计算机系统"')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model'], 'TEST-1')
+        self.assertEqual(parsed['app_raw'], '这句话说明用于计算机系统')
+
+    def test_placeholder_slash_between_semicolons(self):
+        tokens, mode, _ = split_attributes('品牌:MURATA;/;型号:TEST-1')
+        self.assertEqual(tokens, ['品牌:MURATA', '', '型号:TEST-1'])
+        parsed = self.rules.parse('8532241000', '品牌:MURATA;/;型号:TEST-1')
+        self.assertIn('斜杠占位字段', parsed['flags'])
+        self.assertEqual(parsed['model'], 'TEST-1')
+
+    def test_pure_slash_status_fields_and_fraction(self):
+        text = '4/0/计算机用/MURATA/TEST-1/1/10W'
+        tokens, mode, _ = split_attributes(text)
+        self.assertEqual(mode, 'slash_inferred')
+        self.assertEqual(tokens[-1], '1/10W')
+        parsed = self.rules.parse('8532241000', text)
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model'], 'TEST-1')
+        self.assertIn('纯斜杠分段待核查', parsed['flags'])
+
+    def test_pure_slash_brand_or_no_brand(self):
+        text = 'MURATA/TEST-1/计算机用'
+        parsed = self.rules.parse('8532241000', text)
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model'], 'TEST-1')
+        self.assertIn('纯斜杠分段待核查', parsed['flags'])
+        parsed = self.rules.parse('8532241000', '无品牌/TEST-1/计算机用')
+        self.assertEqual(parsed['brand'], '')
+        self.assertEqual(parsed['model'], 'TEST-1')
+        self.assertIn('无品牌标记', parsed['flags'])
+        parsed = self.rules.parse('8532241000', '无中文品牌/MURATA/TEST-1')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model'], 'TEST-1')
+
+    def test_fraction_or_slash_model_is_not_split(self):
+        parsed = self.rules.parse('8532241000', '品牌:MURATA;型号:1/10W')
+        self.assertEqual(parsed['model'], '1/10W')
+        self.assertNotIn('多型号取首个', parsed['flags'])
+        parsed = self.rules.parse('8532241000', '品牌:MURATA;型号:TEST-1/TEST-2')
+        self.assertEqual(parsed['model'], 'TEST-1/TEST-2')
+
+    def test_underscore_as_word_gap_without_changing_model_identifier(self):
+        parsed = self.rules.parse('8532241000', 'MURATA_TEST-1')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model'], 'TEST-1')
+
+    def test_repeated_brand_candidates(self):
+        same = self.rules.parse('8532241000', '品牌:MURATA,MURATA;型号:TEST-1')
+        self.assertEqual(same['brand'], 'MURATA')
+        different = self.rules.parse('8532241000', '品牌:MURATA,SAMSUNG;型号:TEST-1')
+        self.assertEqual(different['brand'], '')
+        self.assertIn('品牌映射冲突', different['flags'])
+
+    def test_chinese_status_with_known_schema_anchor(self):
+        self.rules.schema['SYNTH_STATUS'] = ['品牌类型', '出口享惠情况', '品牌', '型号']
+        parsed = self.rules.parse('SYNTH_STATUS', '同意|同意|MURATA|TEST-1')
+        self.assertEqual(parsed['method'], '顺序提取')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model'], 'TEST-1')
+
+    def test_missing_optional_tail_with_brand_anchor(self):
+        self.rules.schema['SYNTH_TAIL'] = [
+            '品牌类型', '出口享惠情况', '品牌', '型号', 'GTIN', 'CAS', '其他']
+        parsed = self.rules.parse('SYNTH_TAIL', '4|0|MURATA|TEST-1')
+        self.assertEqual(parsed['method'], '顺序提取')
+        self.assertEqual(parsed['model'], 'TEST-1')
+        self.assertIn('模板末尾字段缺失待核查', parsed['flags'])
+        parsed = self.rules.parse('SYNTH_TAIL', '4|0|MURATA|TEST-1||||')
+        self.assertEqual(parsed['method'], '顺序提取')
+
+    def test_numbered_labels_do_not_invent_brand_or_model(self):
+        parsed = self.rules.parse('8532241000', '3|0、品牌类型:xxx;1、品名:?xxx')
+        self.assertEqual(parsed['brand'], '')
+        self.assertEqual(parsed['model'], '')
+
+    def test_empty_brand_slot_keeps_model_despite_placeholders(self):
+        # 品牌位留空、GTIN/CAS 写“无”时,不能因为“无”在别名表里就把型号一起丢掉
+        parsed = self.rules.parse('8532241000', '0|0|多层片式|瓷介||TEST-1|无|无|无')
+        self.assertEqual(parsed['method'], '顺序提取')
+        self.assertEqual(parsed['brand'], '')
+        self.assertEqual(parsed['model_raw'], 'TEST-1')
+        self.assertIn('品牌未识别', parsed['flags'])
+
+    def test_position_brand_conflict_still_falls_back(self):
+        # 品牌位是不认识的写法,但后面有已知品牌 → 仍应转缺项识别并取到 MURATA
+        parsed = self.rules.parse('8532241000', '0|0|多层片式|瓷介|不存在的牌子|TEST-1|MURATA|无|无')
+        self.assertEqual(parsed['method'], '缺项识别')
+        self.assertIn('顺序位置与品牌线索冲突,转缺项识别', parsed['flags'])
+        self.assertEqual(parsed['brand'], 'MURATA')
+
+    def test_label_synonyms_lock_brand_and_model(self):
+        parsed = self.rules.parse('8532241000', '型号规格:TEST-1|品牌名称:MURATA')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model_raw'], 'TEST-1')
+        self.assertIn('品牌标签锁定', parsed['flags'])
+
+    def test_no_model_marker_is_recognized(self):
+        parsed = self.rules.parse('8532241000', '0|0|多层片式|瓷介|MURATA|无型号|无|无|无')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model_raw'], '')
+        self.assertIn('无型号标记', parsed['flags'])
+        self.assertNotIn('型号未识别', parsed['flags'])
+
+    def test_no_brand_and_no_model_markers(self):
+        parsed = self.rules.parse('8532241000', '型号:无;品牌:无中文品牌')
+        self.assertIn('无品牌标记', parsed['flags'])
+        self.assertIn('无型号标记', parsed['flags'])
+        self.assertNotIn('品牌未识别', parsed['flags'])
+        self.assertNotIn('型号未识别', parsed['flags'])
+
+    def test_label_brand_inside_sentence(self):
+        parsed = self.rules.parse('8532241000', '这是一颗贴片电容,品牌:TDK,用于手机')
+        self.assertEqual(parsed['brand'], 'TDK')
+        self.assertIn('品牌标签锁定', parsed['flags'])
+
+    def test_multiple_model_labels_take_first(self):
+        parsed = self.rules.parse('8542321010', '品牌:三星;型号:K4B4G1646D-BCK0;规格型号:DDR4 4GB')
+        self.assertEqual(parsed['brand'], 'SAMSUNG')
+        self.assertEqual(parsed['model_raw'], 'K4B4G1646D-BCK0')
+        self.assertIn('多型号取首个', parsed['flags'])
+
+
+class LabelVariantTests(unittest.TestCase):
+    """人眼一眼能读出、但旧逻辑取不到的写法(键值变体 / 位置变体)。"""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx',
+                          ROOT / 'rules/进出口数据字段需求20260819.xlsx')
+
+    def pair(self, hs, raw):
+        parsed = self.rules.parse(hs, raw)
+        return parsed['brand'], parsed['model_raw']
+
+    def test_key_and_value_in_separate_fields(self):
+        self.assertEqual(self.pair('8532241000', '品牌|MURATA|型号|GRM188R71H104KA93D'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+        self.assertEqual(self.pair('8542321010', '品牌|三星|型号|K4B4G1646D-BCK0'),
+                         ('SAMSUNG', 'K4B4G1646D-BCK0'))
+
+    def test_label_inside_parentheses(self):
+        self.assertEqual(self.pair('8532241000', '(品牌)MURATA(型号)GRM188R71H104KA93D'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+
+    def test_value_before_label(self):
+        self.assertEqual(self.pair('8532241000', 'MURATA(品牌)、GRM188R71H104KA93D(型号)'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+
+    def test_connective_and_separator_variants(self):
+        self.assertEqual(self.pair('8532241000', '品牌为MURATA,型号为GRM188R71H104KA93D'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+        self.assertEqual(self.pair('8532241000', '品牌、村田、型号、GRM188R71H104KA93D'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+        self.assertEqual(self.pair('8532241000', '型号"GRM188R71H104KA93D" 品牌"村田"'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+
+    def test_model_written_before_brand(self):
+        self.assertEqual(self.pair('8532241000', 'GRM188R71H104KA93D MURATA'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+        self.assertEqual(self.pair('8542321010', 'K4B4G1646D-BCK0 三星'),
+                         ('SAMSUNG', 'K4B4G1646D-BCK0'))
+
+    def test_brand_synonyms(self):
+        self.assertEqual(self.pair('8532241000', '制造商:村田制作所 规格:GRM188R71H104KA93D'),
+                         ('MURATA', 'GRM188R71H104KA93D'))
+
+    def test_parameters_are_never_taken_as_model(self):
+        # 反向保护:包装尺寸、容值、用途不能被当成型号
+        for raw in ('品牌:MURATA;规格:0402',
+                    '品牌:MURATA;规格:100nF',
+                    '品牌:MURATA;用途:5G基站'):
+            brand, model = self.pair('8532241000', raw)
+            self.assertEqual(brand, 'MURATA', raw)
+            self.assertEqual(model, '', raw)
+
+    def test_deng_suffix_is_not_part_of_brand_or_model(self):
+        # 申报要素把“这个品牌/型号以及同类”写成 XXX等牌 / XXX等型
+        parsed = self.rules.parse('8532241000', 'MURATA等牌|GRM188R71H104KA93D等型')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model_raw'], 'GRM188R71H104KA93D')
+        self.assertIn('品牌带“等”后缀', parsed['flags'])
+        self.assertIn('多型号取首个', parsed['flags'])
+        parsed = self.rules.parse('8532241000', '4|3|村田等牌|GRM188R71H104KA93D等型')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model_raw'], 'GRM188R71H104KA93D')
+
+    def test_missing_brand_is_named_in_flags(self):
+        # 品牌表里没有 ABACUS,要让用户一眼看出“该补哪一行”
+        parsed = self.rules.parse('8532241000', '4|3|ABACUS等牌|A5E30295495R2W等型')
+        self.assertEqual(parsed['brand'], '')
+        self.assertIn('疑似缺品牌:ABACUS', parsed['flags'])
+
+    def test_model_survives_unresolved_brand_field(self):
+        # 品牌认不出,但“品牌字段后面那一段”明显是型号,不能连型号一起丢
+        parsed = self.rules.parse('8532241000', '4|3|225mW|PANJIT牌/强茂牌|MMBT3904_R1_000Z')
+        self.assertEqual(parsed['brand'], '')
+        self.assertEqual(parsed['model_raw'], 'MMBT3904_R1_000Z')
+        self.assertIn('品牌字段后型号推断', parsed['flags'])
+        self.assertIn('疑似缺品牌:PANJIT', parsed['flags'])
+
+    def test_catch_all_label_is_not_reached_by_typo(self):
+        # 规则表里有一行错写 PANJJIT → Others;PANJIT 不能靠拼写近似落到 Others 上
+        parsed = self.rules.parse('8532241000', '4|3|225mW|PANJIT牌|MMBT3904_R1_000Z')
+        self.assertNotEqual(parsed['brand'], 'Others')
+
+
+class ReverseContainmentTests(unittest.TestCase):
+    """规则表的申报内容是一长串时,G_MODEL 里只写其中一段也要能认。"""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx',
+                          ROOT / 'rules/进出口数据字段需求20260819.xlsx')
+
+    def test_long_rule_containing_the_name_matches(self):
+        # 表里有 “村田制作所(MURATA)” 这类长写法,G_MODEL 写“村田制作所”要能对上
+        self.assertEqual(self.rules.reverse_brand_labels('村田制作所'), frozenset({'MURATA'}))
+
+    def test_multi_brand_rule_is_not_used_as_evidence(self):
+        # “NEXPERIA、DIODES、PANJIT/无中文名称” 这种多品牌混写不能把 PANJIT 认成 Nexperia
+        self.assertEqual(self.rules.reverse_brand_labels('PANJIT'), frozenset())
+        parsed = self.rules.parse('8532241000', '4|3|225mW|PANJIT牌/强茂牌|MMBT3904_R1_000Z8')
+        self.assertNotIn(parsed['brand'], ('Nexperia', 'Onsemi', 'Others'))
+
+    def test_unknown_name_stays_unknown(self):
+        self.assertEqual(self.rules.reverse_brand_labels('ABACUS'), frozenset())
+        self.assertEqual(self.rules.reverse_brand_labels('MMBT3904'), frozenset())
+
+
+if __name__ == '__main__':
+    unittest.main()

+ 74 - 0
tools-service/tests/test_legacy_real_formats.py

@@ -0,0 +1,74 @@
+"""真实海关 G_MODEL 格式的回归用例。
+
+这些形态来自带回的 10,000 条真实样本(结构保留、内容脱敏),
+用于锁定解析器对不规范写法的处理,避免以后改规则时退化。
+"""
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from toolsvc.customs.engine.rules import Rules
+
+
+ROOT = Path(__file__).resolve().parents[1] / 'toolsvc' / 'customs'
+
+
+class RealFormatTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx',
+                          ROOT / 'rules/进出口数据字段需求20260819.xlsx')
+
+    def parse(self, hs, text):
+        return self.rules.parse(hs, text)
+
+    def test_brand_with_chinese_and_latin_parentheses(self):
+        parsed = self.parse('8532241000', '4|3|片式|瓷介|村田(MURATA)|TEST-1|无|无|无')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model_raw'], 'TEST-1')
+        self.assertIn('品牌括号或枚举取词', parsed['flags'])
+
+    def test_brand_label_variants_merge_instead_of_conflict(self):
+        parsed = self.parse('8532241000', '4|3|片式|瓷介|品牌:MICRON;型号:TEST-1')
+        self.assertEqual(parsed['brand'], 'MICRON')
+        self.assertEqual(parsed['model_raw'], 'TEST-1')
+        self.assertNotIn('品牌映射冲突', parsed['flags'])
+
+    def test_placeholder_is_not_taken_as_brand(self):
+        parsed = self.parse('8532241000', '4|3|片式|瓷介||TEST-1||无|无')
+        self.assertEqual(parsed['brand'], '')
+
+    def test_underscore_separated_record(self):
+        parsed = self.parse('8532241000', '0_0_多层片式_瓷介_MURATA_TEST-1')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model_raw'], 'TEST-1')
+        self.assertIn('下划线分段待核查', parsed['flags'])
+
+    def test_space_separated_record(self):
+        parsed = self.parse('8532241000', '0 0 多层片式 瓷介 MURATA TEST-1')
+        self.assertEqual(parsed['brand'], 'MURATA')
+        self.assertEqual(parsed['model_raw'], 'TEST-1')
+        self.assertIn('空格分段待核查', parsed['flags'])
+
+    def test_slash_record_starting_with_chinese_brand_type(self):
+        parsed = self.parse('8532241000', '境内自主品牌/2/多层非片式/瓷介/芯天下/GRM31CR61A226KE19L/无/无')
+        self.assertEqual(parsed['brand'], 'XTX')
+        self.assertEqual(parsed['model_raw'], 'GRM31CR61A226KE19L')
+        self.assertIn('纯斜杠分段待核查', parsed['flags'])
+
+    def test_application_from_usage_slot_when_function_misleading(self):
+        parsed = self.parse('8542321010',
+                            '3|0|手机|功能:读写|未封装|已切割|品牌:海力士(SKHYNIX)|型号:TEST-1|量产||')
+        self.assertEqual(parsed['brand'], 'SK HYNIX')
+        self.assertEqual(parsed['app'], '消费电子')
+
+    def test_application_from_usage_slot_with_parenthesis_brand(self):
+        parsed = self.parse('8542321010',
+                            '4|0|服务器|存储|已封装|已切割|美光(MICRON)|TEST-1|量产|4GB|无|无|无')
+        self.assertEqual(parsed['brand'], 'MICRON')
+        self.assertEqual(parsed['app'], '先进计算')
+
+
+if __name__ == '__main__':
+    unittest.main()

+ 2 - 1
tools-service/tests/test_objects.py

@@ -85,7 +85,8 @@ def test_data_read_aggregate_mode(raw_object):
     assert result["aggregate"] is True
     assert result["aggregate"] is True
     assert result["rows"]
     assert result["rows"]
     assert set(result["columns"]) == {"HS_CODE", "amount", "quantity"}
     assert set(result["columns"]) == {"HS_CODE", "amount", "quantity"}
-    amounts = [row["values"]["amount"] for row in result["rows"]]
+    from decimal import Decimal
+    amounts = [Decimal(row["values"]["amount"]) for row in result["rows"]]
     assert amounts == sorted(amounts, reverse=True)
     assert amounts == sorted(amounts, reverse=True)
 
 
 
 

+ 1 - 1
tools-service/tests/test_rules.py

@@ -87,7 +87,7 @@ def test_port_lookup_returns_name_and_country(rules):
 
 
 def test_unknown_hs_is_not_capacitor(rules):
 def test_unknown_hs_is_not_capacitor(rules):
     parsed = rules.parse("8517700000", "4|0|计算机系统|数据存储|已封装|已切割|MURATA|X1|量产|8G|||")
     parsed = rules.parse("8517700000", "4|0|计算机系统|数据存储|已封装|已切割|MURATA|X1|量产|8G|||")
-    assert parsed["c1"] == "存储器" or parsed["c1"] == "其他"
+    assert parsed["c1"] == ""  # 范围外 HS 不应被误标为存储器
 
 
 
 
 def test_memory_hs_set_is_ten_digits():
 def test_memory_hs_set_is_ten_digits():

+ 3 - 3
tools-service/toolsvc/customs/contract.py

@@ -31,8 +31,8 @@ class CustomsOptions:
     end_year: int = 2026
     end_year: int = 2026
     cutoff_date: str | None = None
     cutoff_date: str | None = None
     directions: list[str] = field(default_factory=lambda: ["I"])
     directions: list[str] = field(default_factory=lambda: ["I"])
-    encoding: str = "utf-8-sig"
-    delimiter: str = ","
+    encoding: str = "auto"
+    delimiter: str = "auto"
     country_code_system: str = "ISO"
     country_code_system: str = "ISO"
     confirmed_complete_years: list[int] = field(default_factory=list)
     confirmed_complete_years: list[int] = field(default_factory=list)
     allow_partial_year_basket: bool = True
     allow_partial_year_basket: bool = True
@@ -48,7 +48,7 @@ class CustomsOptions:
             raise ValueError("country_code_system 只支持 ISO/GEC")
             raise ValueError("country_code_system 只支持 ISO/GEC")
         if not self.directions or set(self.directions) - {"I", "E"}:
         if not self.directions or set(self.directions) - {"I", "E"}:
             raise ValueError("directions 只支持 I / E")
             raise ValueError("directions 只支持 I / E")
-        if self.delimiter not in (",", "\t", ";", "|"):
+        if self.delimiter not in ("auto", ",", "\t", ";", "|"):
             raise ValueError("只支持 , \\t ; | 四种分隔符")
             raise ValueError("只支持 , \\t ; | 四种分隔符")
 
 
     def to_dict(self) -> dict[str, Any]:
     def to_dict(self) -> dict[str, Any]:

+ 104 - 0
tools-service/toolsvc/customs/engine/brandmatch.py

@@ -0,0 +1,104 @@
+"""Indexed fuzzy brand matching. Scores rank evidence, not probabilities."""
+import re
+import unicodedata
+from collections import defaultdict
+from functools import lru_cache
+
+
+def compact(value):
+    return re.sub(r'[^A-Z0-9\u3400-\u9fff]', '',
+                  unicodedata.normalize('NFKC', str(value or '')).upper())
+
+
+def grams(text, size=2):
+    return {text[i:i + size] for i in range(len(text) - size + 1)}
+
+
+def distance(a, b, limit):
+    """Bounded Levenshtein distance, including insertions and deletions."""
+    if abs(len(a) - len(b)) > limit:
+        return limit + 1
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a, 1):
+        row = [i]
+        for j, cb in enumerate(b, 1):
+            row.append(min(row[-1] + 1, prev[j] + 1, prev[j - 1] + (ca != cb)))
+        if min(row) > limit:
+            return limit + 1
+        prev = row
+    return prev[-1]
+
+
+class BrandMatcher:
+    def __init__(self, aliases, excluded, allow_contains=True):
+        self.allow_contains = allow_contains
+        self.names = defaultdict(set)
+        self.index = defaultdict(set)
+        excluded = {compact(v) for v in excluded}
+        for alias, labels in aliases.items():
+            key = compact(alias)
+            if not key or key.isdigit() or key in excluded:
+                continue
+            self.names[key].update(labels)
+        for key in self.names:
+            for gram in grams(key):
+                self.index[gram].add(key)
+
+    @lru_cache(maxsize=32768)
+    def match(self, variants, allow_typo=True):
+        scores = {}
+        for variant in variants:
+            query = compact(variant)
+            if not query or len(query) > 160:
+                continue
+            query_grams = grams(query)
+            candidates = set()
+            if query in self.names:
+                candidates.add(query)
+            for gram in query_grams:
+                candidates.update(self.index.get(gram, ()))
+            for key in candidates:
+                score, method = 0, ''
+                if key == query:
+                    score, method = 100.0, '标点归一'
+                elif key in query:
+                    chinese = bool(re.search(r'[\u3400-\u9fff]', key))
+                    minimum = 2 if chinese else 3
+                    if len(key) < minimum:
+                        continue
+                    # Latin short names must be separate words, never inside model IDs.
+                    pattern = r'[^A-Z0-9\u3400-\u9fff]*'.join(map(re.escape, key))
+                    original = unicodedata.normalize('NFKC', variant).upper()
+                    boundary = re.search(r'(?<![A-Z0-9])' + pattern + r'(?![A-Z0-9])', original)
+                    similarity = len(key) / len(query)
+                    if boundary and similarity >= .8:
+                        score, method = 100 * similarity, '包含匹配'
+                    elif boundary and (chinese or len(key) >= 4) and self.allow_contains:
+                        # 覆盖率不足 80% 时的兜底:整段里出现某个品牌名/别名即算命中,
+                        # 例如“日本兴亚”包含“兴亚”。拉丁名要求至少 4 个字符,
+                        # 避免 ST/AP 这类两三字母缩写在大段文本里误命中。
+                        score, method = 100 * similarity, '子串包含'
+                elif allow_typo and 4 <= len(query) <= 64 and len(key) >= 4:
+                    limit = int(max(len(query), len(key)) * .2 + 1e-9)
+                    if abs(len(key) - len(query)) > limit:
+                        continue
+                    # Any edit removes at most two shared bigrams.
+                    if len(query_grams & grams(key)) < max(1, len(query_grams) - 2 * limit):
+                        continue
+                    d = distance(query, key, limit)
+                    similarity = 1 - d / max(len(query), len(key))
+                    if d <= limit and similarity >= .8:
+                        score, method = 100 * similarity, '拼写近似'
+                if score:
+                    for label in self.names[key]:
+                        candidate = (score, key, method)
+                        current = scores.get(label)
+                        if current is None or candidate > current:
+                            scores[label] = candidate
+        ranked = sorted(((label, *value) for label, value in scores.items()),
+                        key=lambda x: (-x[1], x[0]))
+        if not ranked:
+            return (), '', '', 0.0, ()
+        top = ranked[0]
+        winners = tuple(row[0] for row in ranked if abs(row[1] - top[1]) < 1e-9)
+        return winners, top[2], top[3], top[1], tuple(ranked[:5])

+ 14 - 2
tools-service/toolsvc/customs/engine/numbers.py

@@ -8,8 +8,12 @@
 
 
 from __future__ import annotations
 from __future__ import annotations
 
 
-from decimal import Decimal, InvalidOperation, getcontext
+import re
+import unicodedata
 
 
+from decimal import Decimal, DefaultContext, InvalidOperation, getcontext
+
+DefaultContext.prec = 50  # 新建的处理/接口线程也必须继承 50 位精度
 getcontext().prec = 50
 getcontext().prec = 50
 
 
 D = Decimal
 D = Decimal
@@ -31,7 +35,15 @@ def number(raw: object) -> Decimal | None:
     if isinstance(raw, (int, float)):
     if isinstance(raw, (int, float)):
         value = D(str(raw))
         value = D(str(raw))
         return value if value.is_finite() else None
         return value if value.is_finite() else None
-    text = str(raw).strip().replace(",", "").replace(",", "")
+    text = unicodedata.normalize("NFKC", str(raw)).strip().strip('"\'').strip()
+    text = text.replace("−", "-")
+    if re.search(r"人民币|RMB|CNY|[¥¥€£]", text, re.I):
+        return None
+    text = re.sub(r"^(?:USD\s*|\$\s*)|(?:\s*USD|\s*美元)$", "", text, flags=re.I).strip()
+    if text.startswith('(') and text.endswith(')'):
+        text = '-' + text[1:-1]
+    if re.fullmatch(r"[+-]?\d{1,3}(?:[, ]\d{3})+(?:\.\d+)?", text):
+        text = text.replace(',', '').replace(' ', '')
     if text in {"", "-", "--", "/", "NaN", "nan", "NULL", "null"}:
     if text in {"", "-", "--", "/", "NaN", "nan", "NULL", "null"}:
         return None
         return None
     try:
     try:

+ 7 - 1
tools-service/toolsvc/customs/engine/reports.py

@@ -515,7 +515,13 @@ class Reports:
         for row in self.rules.conflicts:
         for row in self.rules.conflicts:
             self.output.row("映射冲突", row)
             self.output.row("映射冲突", row)
         for key, value in self.stats.items():
         for key, value in self.stats.items():
-            self.output.row("数据质量摘要", [key, value, "模拟数据" if self.config.get("demo") else "单文件统计"])
+            if isinstance(value, dict):
+                for detail, count in value.items():
+                    self.output.row("数据质量摘要", [f"{key}.{detail}", count, "已纳入记录的识别情况" if key == "recognition" else "排除原因"])
+            else:
+                if isinstance(value, list):
+                    value = json.dumps(value, ensure_ascii=False)
+                self.output.row("数据质量摘要", [key, value, "模拟数据" if self.config.get("demo") else "单文件统计"])
         for key, value, description in [
         for key, value, description in [
             ("币种", "USD", "按输入美元;不换汇不缩放"),
             ("币种", "USD", "按输入美元;不换汇不缩放"),
             ("数量单位", "成交数量", "待现场核对成交单位"),
             ("数量单位", "成交数量", "待现场核对成交单位"),

Failā izmaiņas netiks attēlotas, jo tās ir par lielu
+ 790 - 309
tools-service/toolsvc/customs/engine/rules.py


+ 50 - 53
tools-service/toolsvc/customs/engine/storage.py

@@ -16,6 +16,10 @@ import csv
 import hashlib
 import hashlib
 import json
 import json
 import sqlite3
 import sqlite3
+import re
+import unicodedata
+from collections import Counter
+from ...source import iter_source
 from collections.abc import Callable, Iterator
 from collections.abc import Callable, Iterator
 from datetime import date
 from datetime import date
 from pathlib import Path
 from pathlib import Path
@@ -48,23 +52,7 @@ DIMENSIONS = [
 REQUIRED = ["I_E_FLAG", "HS_CODE", "YEAR", "WEEK", "MONORSUM", "COUNT", "G_MODEL"]
 REQUIRED = ["I_E_FLAG", "HS_CODE", "YEAR", "WEEK", "MONORSUM", "COUNT", "G_MODEL"]
 
 
 #: 中文/别名表头 → 规范字段名
 #: 中文/别名表头 → 规范字段名
-ALIASES = {
-    "进出口": "I_E_FLAG",
-    "进/出口": "I_E_FLAG",
-    "HS编码": "HS_CODE",
-    "HS 编码": "HS_CODE",
-    "年份": "YEAR",
-    "周": "WEEK",
-    "金额": "MONORSUM",
-    "金额:人民币": "MONORSUM",
-    "金额:美元": "MONORSUM",
-    "数量": "COUNT",
-    "品牌/型号": "G_MODEL",
-    "原产国": "ORIGIN_COUNTRY",
-    "贸易国": "TRADE_AREA_CODE",
-    "启运国": "TRADE_COUNTRY",
-    "经停港/指运港": "DISTINATE_PORT",
-}
+ALIASES = {'MONORSUM': 'MONORSUM', '金额': 'MONORSUM', '金额(美元)': 'MONORSUM', '金额(美元)': 'MONORSUM', '金额:美元': 'MONORSUM', '美元金额': 'MONORSUM', '成交金额': 'MONORSUM', '总金额': 'MONORSUM', '进出口': 'I_E_FLAG', '进/出口': 'I_E_FLAG', '进出口标志': 'I_E_FLAG', '进口/出口': 'I_E_FLAG', '进出口类型': 'I_E_FLAG', 'HS编码': 'HS_CODE', 'HS 编码': 'HS_CODE', '商品编码': 'HS_CODE', '商品编号': 'HS_CODE', '海关编码': 'HS_CODE', '税则号列': 'HS_CODE', '商品编码(10位)': 'HS_CODE', '年份': 'YEAR', '年度': 'YEAR', '周': 'WEEK', '周数': 'WEEK', 'ISO周': 'WEEK', '统计周': 'WEEK', '数量': 'COUNT', '成交数量': 'COUNT', '数量(成交)': 'COUNT', '数量(成交)': 'COUNT', '品牌/型号': 'G_MODEL', '品牌型号': 'G_MODEL', '申报要素': 'G_MODEL', '规格型号': 'G_MODEL', '原产国': 'ORIGIN_COUNTRY', '原产国(地区)': 'ORIGIN_COUNTRY', '贸易国': 'TRADE_AREA_CODE', '贸易国(地区)': 'TRADE_AREA_CODE', '启运国': 'TRADE_COUNTRY', '启运国(地区)': 'TRADE_COUNTRY', '经停港/指运港': 'DISTINATE_PORT', '经停港': 'DISTINATE_PORT', '指运港': 'DISTINATE_PORT', '目的国': 'DESTINATION_COUNTRY', '境内目的地/境内货源地': 'DISTRICT_CODE', '境内目的地': 'DISTRICT_CODE', '贸易方式': 'TRADE_MODE', '申报地海关': 'DECLARING_CUSTOMS', '运输方式': 'TRAF_MODE', '包装种类': 'PACKING_TYPE', 'MONORSUM_USD': 'MONORSUM'}
 
 
 Progress = Callable[[str, float], None]
 Progress = Callable[[str, float], None]
 
 
@@ -86,36 +74,22 @@ def sha256_file(path: str | Path, chunk: int = 1 << 20) -> str:
     return digest.hexdigest()
     return digest.hexdigest()
 
 
 
 
-def _iter_source_rows(
-    source: Path, encoding: str, delimiter: str
-) -> Iterator[tuple[int, list[str], list[str] | None]]:
-    """按统一方式产出 (物理行号, 表头, 行) —— CSV 与 XLSX 都支持。"""
-
-    if source.suffix.lower() in {".xlsx", ".xlsm"}:
-        from openpyxl import load_workbook
-
-        workbook = load_workbook(source, read_only=True, data_only=True)
-        sheet = workbook.worksheets[0]
-        headers: list[str] | None = None
-        for index, row in enumerate(sheet.iter_rows(values_only=True), 1):
-            values = ["" if cell is None else str(cell) for cell in row]
-            if headers is None:
-                headers = values
-                yield index, headers, None
-            else:
-                yield index, headers, values
-        workbook.close()
-        return
-
-    with source.open(encoding=encoding, newline="") as handle:
-        reader = csv.reader(handle, delimiter=delimiter, strict=True)
-        headers: list[str] = []
-        for line_number, row in enumerate(reader, 1):
-            if line_number == 1:
-                headers = row
-                yield line_number, headers, None
-            else:
-                yield line_number, headers, row
+def _iter_source_rows(source: Path, encoding: str, delimiter: str):
+    # Retained for callers using the engine's previous reader helper.
+    for row_id, end_line, headers, values in iter_source(source, encoding, delimiter):
+        yield end_line, headers, values
+
+
+def normalize_header(value):
+    return unicodedata.normalize("NFKC", str(value or "")).replace("\ufeff", "").replace("\u200b", "").strip()
+
+
+def normalize_code(value):
+    text = normalize_header(value)
+    # Excel may export integer code cells with a .0 suffix. No rounding or padding.
+    if re.fullmatch(r"[0-9]+\.0+", text):
+        text = text.split('.')[0]
+    return text
 
 
 
 
 def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = None) -> tuple[sqlite3.Connection, dict]:
 def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = None) -> tuple[sqlite3.Connection, dict]:
@@ -141,6 +115,8 @@ def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = No
         "alerts": 0,
         "alerts": 0,
         "min_month": None,
         "min_month": None,
         "max_month": None,
         "max_month": None,
+        "recognition": {},
+        "exclusion_reasons": {},
     }
     }
 
 
     def issue(row_id: int, field: str, raw: object, reason: str) -> None:
     def issue(row_id: int, field: str, raw: object, reason: str) -> None:
@@ -148,10 +124,17 @@ def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = No
         stats["alerts"] += 1
         stats["alerts"] += 1
 
 
     headers_seen: list[str] = []
     headers_seen: list[str] = []
+    recognition = Counter()
+    exclusions = Counter()
     report("读取源文件…", 0.02)
     report("读取源文件…", 0.02)
-    for row_id, headers, values in _iter_source_rows(source, config.get("encoding", "utf-8-sig"), config.get("delimiter", ",")):
+    for row_id, end_line, headers, values in iter_source(source, config.get("encoding", "auto"), config.get("delimiter", "auto")):
         if values is None:
         if values is None:
-            headers_seen = [ALIASES.get(name.strip(), name.strip()) for name in headers]
+            cleaned = [normalize_header(name) for name in headers]
+            lookup = {normalize_header(key).upper(): value for key, value in ALIASES.items()}
+            headers_seen = [lookup.get(name.upper(), name.upper()) for name in cleaned]
+            if "MONORSUM" not in headers_seen and any("人民币" in name or "RMB" in name.upper() or "CNY" in name.upper() for name in cleaned):
+                raise ValueError("发现人民币金额列;本工具统计口径为美元,请先换汇并明确提供 MONORSUM_USD")
+            stats["input_fields"] = headers_seen
             if len(set(headers_seen)) != len(headers_seen):
             if len(set(headers_seen)) != len(headers_seen):
                 raise ValueError("源文件存在重复字段名")
                 raise ValueError("源文件存在重复字段名")
             missing = set(REQUIRED) - set(headers_seen)
             missing = set(REQUIRED) - set(headers_seen)
@@ -166,12 +149,16 @@ def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = No
         else:
         else:
             row = dict(zip(headers_seen, values))
             row = dict(zip(headers_seen, values))
             status = ""
             status = ""
-        hs = (row.get("HS_CODE") or "").strip()
-        direction = {"进口": "I", "出口": "E"}.get((row.get("I_E_FLAG") or "").strip().upper(), (row.get("I_E_FLAG") or "").strip().upper())
+        hs = normalize_code(row.get("HS_CODE"))
+        direction_raw = normalize_code(row.get("I_E_FLAG")).upper()
+        direction = {"进口": "I", "出口": "E"}.get(direction_raw, direction_raw)
+        if not normalize_header(row.get("G_MODEL")):
+            status = status or "G_MODEL为空"
+
 
 
         thursday: date | None
         thursday: date | None
         try:
         try:
-            thursday = date.fromisocalendar(int(row["YEAR"]), int(row["WEEK"]), 4)
+            thursday = date.fromisocalendar(int(normalize_code(row["YEAR"])), int(normalize_code(row["WEEK"])), 4)
         except (ValueError, TypeError, KeyError):
         except (ValueError, TypeError, KeyError):
             thursday = None
             thursday = None
             status = status or "无效ISO年份或周号"
             status = status or "无效ISO年份或周号"
@@ -182,12 +169,14 @@ def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = No
             status = status or "不在项目HS范围"
             status = status or "不在项目HS范围"
         if direction not in config["directions"]:
         if direction not in config["directions"]:
             status = status or "不在配置进出口范围"
             status = status or "不在配置进出口范围"
+        if thursday and not int(config.get("start_year", 1900)) <= thursday.year <= int(config.get("end_year", 9998)):
+            status = status or "不在配置年份范围"
         if thursday and thursday > cutoff:
         if thursday and thursday > cutoff:
             status = status or "晚于截止日期的周四"
             status = status or "晚于截止日期的周四"
 
 
         database.execute(
         database.execute(
             "INSERT INTO raw VALUES(?,?,?,?)",
             "INSERT INTO raw VALUES(?,?,?,?)",
-            (row_id, row_id + 1, json.dumps(row, ensure_ascii=False), status or "纳入"),
+            (row_id, end_line, json.dumps(row, ensure_ascii=False), status or "纳入"),
         )
         )
         for key in ("MONORSUM", "COUNT"):
         for key in ("MONORSUM", "COUNT"):
             value = number(row.get(key))
             value = number(row.get(key))
@@ -197,9 +186,15 @@ def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = No
                 issue(row_id, key, row.get(key), "零值或负数;保留原值")
                 issue(row_id, key, row.get(key), "零值或负数;保留原值")
         if status:
         if status:
             stats["excluded"] += 1
             stats["excluded"] += 1
+            exclusions[status] += 1
             issue(row_id, "记录范围", hs, status)
             issue(row_id, "记录范围", hs, status)
         else:
         else:
             parsed = rules.parse(hs, row["G_MODEL"])
             parsed = rules.parse(hs, row["G_MODEL"])
+            recognition["brand_resolved" if parsed["brand"] else "brand_unresolved"] += 1
+            recognition["model_present" if parsed["model"] else "model_absent"] += 1
+            for label, marker in [("explicit_no_brand", "无品牌标记"), ("explicit_no_model", "无型号标记"), ("unknown_brand", "疑似缺品牌:"), ("brand_conflict", "品牌映射冲突")]:
+                if marker in parsed.get("flags", ""):
+                    recognition[label] += 1
             port, port_country = rules.port(row.get("DISTINATE_PORT"))
             port, port_country = rules.port(row.get("DISTINATE_PORT"))
             record = dict(
             record = dict(
                 parsed,
                 parsed,
@@ -245,6 +240,8 @@ def ingest(config: dict, rules: Rules, out: Path, progress: Progress | None = No
         database.execute(f"CREATE INDEX idx_{suffix} ON monthly({columns})")
         database.execute(f"CREATE INDEX idx_{suffix} ON monthly({columns})")
     database.commit()
     database.commit()
     stats.update(
     stats.update(
+        recognition=dict(recognition),
+        exclusion_reasons=dict(exclusions),
         sha256=sha256_file(source),
         sha256=sha256_file(source),
         cutoff=str(cutoff),
         cutoff=str(cutoff),
         partial_year_rule="仅 confirmed_complete_years 明确确认为全年,其余均为已提供期间暂算",
         partial_year_rule="仅 confirmed_complete_years 明确确认为全年,其余均为已提供期间暂算",

+ 278 - 0
tools-service/toolsvc/customs/engine/structure.py

@@ -0,0 +1,278 @@
+"""Conservative structural reading of irregular declaration-element text."""
+
+import re
+from functools import lru_cache
+
+
+PRIMARY = set("||;;\t\r\n")
+QUOTES = {'"': '"', "'": "'", '“': '”', '‘': '’'}
+EMPTY_MARKERS = {'/', '/'}
+NO_BRAND = re.compile(r'^(?:无品牌|无牌|无厂牌|无商标|无品牌信息|无品牌标识|无品牌名称)$')
+BRAND_QUALIFIER = re.compile(r'^(?:无中文品牌|无外文品牌|无中文名称|无外文名称|无中文商标|无外文商标)$')
+# “无型号/无规格”这类显式缺项,属于“已经说清楚没有”,不算识别失败。
+NO_MODEL = re.compile(
+    r'^(?:无型号|无中文型号|无外文型号|无型号信息|无具体型号|无型号标识|无牌号|无规格|无规格型号|'
+    r'无货号|无零件号|未提供型号|未提供规格|不适用|无|—|–|-|/|/|N/?A|NA|NONE|NULL|空)$', re.I)
+# 标签:品牌/型号的常见写法都算线索,长写法放在前面避免被短写法截断。
+LABEL = re.compile(
+    r'(?<!\w)(品牌名称|外文品牌|中文品牌|品牌商标|品牌|厂牌|商标|制造商|生产商|生产厂家|'
+    r'型号规格|规格型号|产品型号|型号编号|型号|料号|物料编码|物料号|零件号|货号|规格|'
+    r'用途|应用领域|应用场景|功能|'
+    r'BRAND|MODEL|USAGE|FUNCTION)'
+    r'\s*(?:[::=、]|为|是|["“”])\s*', re.I,
+)
+# 把同义标签归一到两个主类,后续只认“品牌/型号”。
+LABEL_ALIAS = {
+    '品牌名称': '品牌', '外文品牌': '品牌', '中文品牌': '品牌', '品牌商标': '品牌',
+    '厂牌': '品牌', '商标': '品牌', '制造商': '品牌', '生产商': '品牌', '生产厂家': '品牌',
+    '型号规格': '型号', '规格型号': '型号', '产品型号': '型号', '型号编号': '型号',
+    '料号': '型号', '物料编码': '型号', '物料号': '型号', '零件号': '型号', '货号': '型号',
+    '规格': '型号',
+}
+# “规格”这类弱标签只在一个值明显像型号时才采用,避免把 0402、100nF 这类参数当成型号。
+WEAK_LABELS = {'规格'}
+# 括号里写标签的两种写法:“(品牌)MURATA” 与 “MURATA(品牌)”。
+PAREN_LABEL = re.compile(
+    r'[((]\s*(品牌名称|外文品牌|中文品牌|品牌|厂牌|商标|制造商|生产商|'
+    r'型号规格|规格型号|产品型号|型号|料号|物料编码|物料号|零件号|货号|规格)'
+    r'\s*[))]', re.I)
+# 一个值前后残留的分隔符/引号/括号。
+TRIM_CHARS = ' \t\u3000_、,,||;;:://"\'“”‘’()()【】[]'
+# 值里混进了另一个标签,说明取值越界了,从这里截断。
+INNER_LABEL = re.compile(r'(?<!\w)(品牌名称|外文品牌|中文品牌|品牌|厂牌|商标|'
+                         r'型号规格|规格型号|产品型号|型号|料号|物料编码|物料号|零件号|货号|规格)')
+NUMBERED = re.compile(r'^\s*\d{1,2}\s*[、..]\s*')
+STATUS = re.compile(r'^[0-4]$')
+CHINESE_STATUS = re.compile(r'^[\u3400-\u9fff]{1,12}$')
+
+
+def unwrap_outer_quotes(value):
+    text = str(value or '').strip()
+    while len(text) >= 2 and text[0] in QUOTES and text[-1] == QUOTES[text[0]]:
+        quote = text[0]
+        text = text[1:-1].strip()
+        if quote in ('"', "'"):
+            text = text.replace(quote * 2, quote)
+    return text
+
+
+def _primary_tokens(text):
+    parts, buffer = [], []
+    quote = None
+    bracket = 0
+    for char in text:
+        if quote:
+            buffer.append(char)
+            if char == quote:
+                quote = None
+            continue
+        if char in QUOTES:
+            quote = QUOTES[char]
+            buffer.append(char)
+        elif char in '((':
+            bracket += 1
+            buffer.append(char)
+        elif char in '))' and bracket:
+            bracket -= 1
+            buffer.append(char)
+        elif char in PRIMARY and not bracket:
+            parts.append(''.join(buffer).strip())
+            buffer = []
+        else:
+            buffer.append(char)
+    parts.append(''.join(buffer).strip())
+    return parts
+
+
+def _slash_tokens(text):
+    parts, buffer = [], []
+    for index, char in enumerate(text):
+        if char not in '//':
+            buffer.append(char)
+            continue
+        before = text[index - 1] if index else ''
+        after = text[index + 1] if index + 1 < len(text) else ''
+        current = ''.join(buffer).strip()
+        # The first status pair can use slash; later numeric fractions stay whole.
+        first_status_pair = not parts and STATUS.fullmatch(current) and bool(after) and after in '01234'
+        if before.isdigit() and after.isdigit() and current.isdigit() and not first_status_pair:
+            buffer.append(char)
+        else:
+            parts.append(current)
+            buffer = []
+    parts.append(''.join(buffer).strip())
+    return parts
+
+
+def _underscore_tokens(text):
+    parts = [part.strip() for part in text.split('_')]
+    return parts if any(parts) else [text]
+
+
+def _space_tokens(text):
+    parts = [part for part in re.split(r'\s{1,}', text.strip()) if part]
+    return parts
+
+
+def split_attributes(raw, is_brand=None):
+    """Return candidate fields, split mode, and a structure warning if needed."""
+    text = unwrap_outer_quotes(raw)
+    parts = _primary_tokens(text)
+    mode = 'primary'
+    warning = ''
+    labelled = bool(LABEL.search(NUMBERED.sub('', text)))
+    if len(parts) == 1 and text.count('_') >= 2 and not labelled:
+        proposed = _underscore_tokens(text)
+        # 下划线在真实数据里常被当作字段分隔;只有在能得到多个非空段时才采用。
+        if len(proposed) >= 3 and sum(bool(x) for x in proposed) >= 2:
+            parts, mode, warning = proposed, 'underscore_inferred', '下划线分段待核查'
+    if len(parts) == 1 and text.count('_') < 2 and not labelled:
+        proposed = _space_tokens(text)
+        # 空格只在明显是多字段时才切,避免拆散 "AP MEMORY" 这类品牌名。
+        if len(proposed) >= 3:
+            parts, mode, warning = proposed, 'space_inferred', '空格分段待核查'
+    if len(parts) == 1 and text.count('/') + text.count('/') >= 2:
+        proposed = _slash_tokens(text)
+        first_two_status = (len(proposed) >= 3 and
+                            STATUS.fullmatch(proposed[0]) and STATUS.fullmatch(proposed[1]))
+        first_two_chinese = (len(proposed) >= 5 and
+                             CHINESE_STATUS.fullmatch(proposed[0]) and
+                             CHINESE_STATUS.fullmatch(proposed[1]))
+        # “境内自主品牌/2/…” 这类中文品牌类型 + 数字状态,也是可靠的结构线索。
+        chinese_then_status = (len(proposed) >= 3 and
+                               CHINESE_STATUS.fullmatch(proposed[0]) and
+                               STATUS.fullmatch(proposed[1]))
+        # 前几个字段里出现已知品牌,说明这确实是一条属性串而不是自由文本。
+        early_brand = (is_brand is not None and
+                       any(is_brand(part) for part in proposed[1:5] if part))
+        labelled = sum(bool(LABEL.search(NUMBERED.sub('', part))) for part in proposed) >= 2
+        first_brand = (len(proposed) >= 3 and
+                       (NO_BRAND.fullmatch(proposed[0]) or
+                        (is_brand is not None and is_brand(proposed[0])) or
+                        (BRAND_QUALIFIER.fullmatch(proposed[0]) and
+                         is_brand is not None and is_brand(proposed[1]))))
+        if first_two_status or first_two_chinese or chinese_then_status or early_brand or labelled or first_brand:
+            parts, mode, warning = proposed, 'slash_inferred', '纯斜杠分段待核查'
+    parts = [unwrap_outer_quotes(part) if part not in EMPTY_MARKERS else '' for part in parts]
+    return parts, mode, warning
+
+
+def clean_value(value, separator=''):
+    """把一个标签取值修干净:去掉两端残留的分隔符/引号,并在混入另一个标签处截断。"""
+    text = str(value or '').strip(TRIM_CHARS)
+    inner = INNER_LABEL.search(text)
+    if inner:
+        text = text[:inner.start()].strip(TRIM_CHARS)
+    if separator == '、':
+        # 用顿号做键值分隔的记录,值本身也在顿号处结束
+        text = re.split(r'[、]', text, maxsplit=1)[0].strip(TRIM_CHARS)
+    return text
+
+
+# 键和值被主分隔符切开、各自占一段的写法:品牌|MURATA|型号|X
+KEY_TOKEN = re.compile(
+    r'^(?:品牌名称|外文品牌|中文品牌|品牌|厂牌|商标|制造商|生产商|生产厂家|'
+    r'型号规格|规格型号|产品型号|型号|料号|物料编码|物料号|零件号|货号|BRAND|MODEL)$', re.I)
+# 走正则之前先做一次子串判断:不含任何标签词的段直接跳过,省掉三次正则扫描。
+# 只放"真出现就可能是标签"的词,像“型”“牌”这种到处都有的字不放,否则等于没筛。
+LABEL_HINTS = ('品牌', '型号', '厂牌', '商标', '制造', '生产', '料号', '物料', '零件', '货号',
+               '规格', '用途', '应用', '功能')
+LATIN_HINTS = ('BRAND', 'MODEL', 'USAGE', 'FUNCTION')
+
+
+def label_kind(name):
+    """同义标签归一:品牌系列 → 品牌,型号系列 → 型号,其余原样大写。"""
+    mapped = LABEL_ALIAS.get(name, LABEL_ALIAS.get(str(name).upper(), name))
+    return str(mapped).upper()
+
+
+def looks_like_model(text):
+    """一段文字是否明显像料号:够长、有字母也有数字。
+
+    用来替代"靠位置猜型号",所以门槛保守——100nF、0402、DDR4、5G基站 都不算。
+    """
+    value = str(text or '')
+    return (len(value) >= 6 and len(re.findall(r'[A-Za-z]', value)) >= 2
+            and len(re.findall(r'\d', value)) >= 3)
+
+
+# 申报要素常把“这个品牌/型号以及同类”写成“XXX等牌”“XXX等型”,
+# 后缀不是名字的一部分,留着会把同一个品牌/型号劈成多条。
+BRAND_SUFFIXES = ('等品牌', '等商标', '等厂牌', '等厂商', '等牌', '等', '牌', '商标')
+MODEL_SUFFIXES = ('等型号', '等规格', '等牌号', '等型', '等')
+
+
+def strip_brand_suffix(text):
+    value = str(text or '').strip()
+    # “无品牌/无中文品牌”本身就是完整语义,不能把结尾的“牌”当后缀剥掉
+    if NO_BRAND.fullmatch(value) or BRAND_QUALIFIER.fullmatch(value):
+        return value, False
+    for suffix in BRAND_SUFFIXES:
+        if value.endswith(suffix) and len(value) > len(suffix):
+            return value[:-len(suffix)].strip(), True
+    return value, False
+
+
+def strip_model_suffix(text):
+    value = str(text or '').strip()
+    for suffix in MODEL_SUFFIXES:
+        if value.endswith(suffix) and len(value) > len(suffix):
+            return value[:-len(suffix)].strip(), True
+    return value, False
+
+
+def labels_in(text):
+    """Find explicit labels within one candidate field, preserving sentence values.
+
+    支持四种写法:
+      · 键值带分隔符:品牌:MURATA / 品牌为MURATA / 型号"X" / 品牌、村田
+      · 括号里放标签:(品牌)MURATA
+      · 标签放在值后面:MURATA(品牌)
+    弱标签(如“规格”)只有在值看起来像型号时才采用,避免把 0402 当成型号。
+
+    先做一次廉价的子串判断再走正则,并对结果做缓存——解析 400 万条时这是最热的一段。
+    """
+    if not text:
+        return iter(())
+    probe = str(text)
+    upper = probe.upper()
+    if (not any(hint in probe for hint in LABEL_HINTS)
+            and not any(hint in upper for hint in LATIN_HINTS)):
+        return iter(())
+    return iter(_labels_cached(probe))
+
+
+@lru_cache(maxsize=32768)
+def _labels_cached(text):
+    return tuple(_labels_impl(text))
+
+
+def _labels_impl(text):
+    text = NUMBERED.sub('', text)
+    # “(品牌)MURATA” → “品牌:MURATA”,统一交给下面的标签正则处理
+    normalised = PAREN_LABEL.sub(lambda m: f'{m.group(1)}:', text)
+    matches = list(LABEL.finditer(normalised))
+    for index, match in enumerate(matches):
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(normalised)
+        separator = normalised[match.end(1):match.end()].strip()
+        raw = normalised[match.end():end]
+        value = clean_value(raw, '、' if '、' in separator else '')
+        if not value:
+            continue
+        name = match.group(1)
+        if name in WEAK_LABELS and not looks_like_model(value):
+            continue
+        yield label_kind(name), value
+    # “MURATA(品牌)”:标签在值后面,取括号前面那一段
+    for match in PAREN_LABEL.finditer(text):
+        head = text[:match.start()]
+        if PAREN_LABEL.search(head):
+            # 前面已经出现过括号标签,说明这段是被前面的“(品牌)MURATA”形式吃掉的,跳过
+            continue
+        tail = re.split(r'[、,,||;;]', head)[-1]
+        value = clean_value(tail)
+        if value:
+            name = match.group(1)
+            if name in WEAK_LABELS and not looks_like_model(value):
+                continue
+            yield label_kind(name), value

+ 3 - 3
tools-service/toolsvc/customs/pipeline.py

@@ -16,7 +16,7 @@ from .engine.rules import Rules
 from .engine.storage import DIMENSIONS, connect, ingest
 from .engine.storage import DIMENSIONS, connect, ingest
 from .engine import reports as reports_module
 from .engine import reports as reports_module
 
 
-ENGINE_VERSION = "2.0.0"
+ENGINE_VERSION = "2.1.0"
 DEFAULT_RULES = Path(__file__).with_name("rules")
 DEFAULT_RULES = Path(__file__).with_name("rules")
 
 
 
 
@@ -140,8 +140,8 @@ class DefaultCustomsProcessor:
         clause = " AND ".join(f"{key}=?" for key in filters) or "1=1"
         clause = " AND ".join(f"{key}=?" for key in filters) or "1=1"
         columns = ",".join(group_by)
         columns = ",".join(group_by)
         sql = (
         sql = (
-            f'SELECT {columns},SUM(CAST(amount AS REAL)) amount,SUM(CAST(quantity AS REAL)) quantity,COUNT(*) n '
-            f'FROM fact WHERE {clause} GROUP BY {columns} ORDER BY amount DESC'
+            f'SELECT {columns},dsum(amount) amount,dsum(quantity) quantity,COUNT(*) n '
+            f'FROM fact WHERE {clause} GROUP BY {columns} ORDER BY CAST(amount AS REAL) DESC'
         )
         )
         report("执行自定义聚合…", 0.8)
         report("执行自定义聚合…", 0.8)
         output = Output(output_dir)
         output = Output(output_dir)

+ 8 - 32
tools-service/toolsvc/objects.py

@@ -126,7 +126,7 @@ class ObjectStore:
         columns: list[str] = []
         columns: list[str] = []
         rows = 0
         rows = 0
         meta: dict = {}
         meta: dict = {}
-        if index and path.suffix.lower() in {".csv", ".txt", ".xlsx", ".xlsm"}:
+        if index and path.suffix.lower() in {".csv", ".tsv", ".txt", ".xlsx", ".xlsm"}:
             columns, rows = self._index_raw(object_id, path)
             columns, rows = self._index_raw(object_id, path)
         meta["sqlite_path"] = str(directory / "raw.sqlite") if index and columns else ""
         meta["sqlite_path"] = str(directory / "raw.sqlite") if index and columns else ""
 
 
@@ -266,7 +266,7 @@ class ObjectStore:
             "stats": {
             "stats": {
                 key: value
                 key: value
                 for key, value in (result.get("stats") or {}).items()
                 for key, value in (result.get("stats") or {}).items()
-                if key in {"total", "included", "excluded", "alerts", "min_month", "max_month", "seconds", "engine", "cutoff"}
+                if key in {"total", "included", "excluded", "alerts", "min_month", "max_month", "seconds", "engine", "cutoff", "recognition", "exclusion_reasons"}
             },
             },
         }
         }
         # 磁盘占用:把整次运行目录(xlsx + 逐表 CSV + 两个 sqlite + 明细)都算进去
         # 磁盘占用:把整次运行目录(xlsx + 逐表 CSV + 两个 sqlite + 明细)都算进去
@@ -487,39 +487,15 @@ def _media_type(suffix: str) -> str:
 
 
 
 
 def _iter_rows(path: Path):
 def _iter_rows(path: Path):
-    """产出 (表头, 行迭代器)。CSV 与 XLSX 都支持。"""
-
-    suffix = path.suffix.lower()
-    if suffix in {".xlsx", ".xlsm"}:
-        from openpyxl import load_workbook
-
-        workbook = load_workbook(path, read_only=True, data_only=True)
-        sheet = workbook.worksheets[0]
-        iterator = sheet.iter_rows(values_only=True)
-        headers = ["" if value is None else str(value) for value in next(iterator)]
-        return headers, (row for row in iterator)
-    if suffix == ".json":
-        payload = json.loads(path.read_text(encoding="utf-8-sig"))
-        if isinstance(payload, dict):
-            payload = payload.get("rows") or payload.get("data") or []
-        headers = list(payload[0].keys()) if payload else []
-        return headers, (list(item.get(header) for header in headers) for item in payload)
-    encoding = _sniff_encoding(path)
-    handle = path.open(encoding=encoding, newline="")
-    reader = csv.reader(handle)
-    headers = next(reader)
-    return headers, reader
+    from .source import iter_source
+    iterator = iter_source(path)
+    _, _, headers, _ = next(iterator)
+    return headers, (values for _, _, _, values in iterator)
 
 
 
 
 def _sniff_encoding(path: Path) -> str:
 def _sniff_encoding(path: Path) -> str:
-    raw = path.open("rb").read(4096)
-    for encoding in ("utf-8-sig", "utf-8", "gb18030", "utf-16"):
-        try:
-            raw.decode(encoding)
-            return encoding
-        except UnicodeDecodeError:
-            continue
-    return "utf-8"
+    from .source import detect_encoding
+    return detect_encoding(path)
 
 
 
 
 store = ObjectStore()
 store = ObjectStore()

+ 35 - 5
tools-service/toolsvc/readers.py

@@ -14,6 +14,7 @@ import json
 import sqlite3
 import sqlite3
 from pathlib import Path
 from pathlib import Path
 
 
+from .customs.engine.numbers import D, number, to_text
 from .config import settings
 from .config import settings
 from .objects import store
 from .objects import store
 
 
@@ -38,9 +39,36 @@ def _numeric_columns(sqlite_path: Path, table: str) -> set[str]:
     return {name[5:] for name in names if name.startswith("_num_")}
     return {name[5:] for name in names if name.startswith("_num_")}
 
 
 
 
+def _decimal_aggregate(operation):
+    class Aggregate:
+        def __init__(self):
+            self.value, self.count, self.bad = None, 0, False
+
+        def step(self, raw):
+            value = number(raw)
+            if value is None:
+                self.bad = True
+                return
+            self.count += 1
+            if self.value is None:
+                self.value = value
+            elif operation in ('sum', 'avg'):
+                self.value += value
+            else:
+                self.value = (min if operation == 'min' else max)(self.value, value)
+
+        def finalize(self):
+            if self.bad or not self.count:
+                return None
+            return to_text(self.value / D(self.count) if operation == 'avg' else self.value)
+    return Aggregate
+
+
 def _connect(sqlite_path: Path) -> sqlite3.Connection:
 def _connect(sqlite_path: Path) -> sqlite3.Connection:
     connection = sqlite3.connect(str(sqlite_path))
     connection = sqlite3.connect(str(sqlite_path))
     connection.row_factory = sqlite3.Row
     connection.row_factory = sqlite3.Row
+    for operation in ('sum', 'avg', 'min', 'max'):
+        connection.create_aggregate('exact_' + operation, 1, _decimal_aggregate(operation))
     return connection
     return connection
 
 
 
 
@@ -172,6 +200,7 @@ def _aggregate(connection, sqlite_path, object_id, table, sqlite_name, available
         raise ReadError("aggregate 需要 metrics 或 sum")
         raise ReadError("aggregate 需要 metrics 或 sum")
 
 
     selects: list[str] = []
     selects: list[str] = []
+    aliases: list[str] = []
     for column in group_by:
     for column in group_by:
         selects.append(_quote(column))
         selects.append(_quote(column))
     for metric in metrics:
     for metric in metrics:
@@ -184,16 +213,17 @@ def _aggregate(connection, sqlite_path, object_id, table, sqlite_name, available
         else:
         else:
             if column not in available:
             if column not in available:
                 raise ReadError(f"聚合列不存在: {column}")
                 raise ReadError(f"聚合列不存在: {column}")
-            target = f'_num_{column}' if column in numeric else column
-            expression = f"{function.upper()}(CAST({_quote(target)} AS REAL))"
+            expression = (f"COUNT({_quote(column)})" if function == "count"
+                          else f"exact_{function}({_quote(column)})")
         alias = str(metric.get("alias") or f"{function}_{column}")
         alias = str(metric.get("alias") or f"{function}_{column}")
-        selects.append(f'{expression} AS "{alias}"')
+        aliases.append(alias)
+        selects.append(f'{expression} AS {_quote(alias)}')
 
 
     group_sql = f' GROUP BY {",".join(_quote(column) for column in group_by)}' if group_by else ""
     group_sql = f' GROUP BY {",".join(_quote(column) for column in group_by)}' if group_by else ""
     order_sql = ""
     order_sql = ""
     if metrics:
     if metrics:
         first_alias = str(metrics[0].get("alias") or f"{metrics[0].get('fn', 'sum')}_{metrics[0].get('column')}")
         first_alias = str(metrics[0].get("alias") or f"{metrics[0].get('fn', 'sum')}_{metrics[0].get('column')}")
-        order_sql = f' ORDER BY "{first_alias}" DESC'
+        order_sql = f' ORDER BY CAST({_quote(first_alias)} AS REAL) DESC'
 
 
     rows = connection.execute(
     rows = connection.execute(
         f'SELECT {",".join(selects)} FROM {_quote(sqlite_name)} WHERE {where}{group_sql}{order_sql}',
         f'SELECT {",".join(selects)} FROM {_quote(sqlite_name)} WHERE {where}{group_sql}{order_sql}',
@@ -218,7 +248,7 @@ def _aggregate(connection, sqlite_path, object_id, table, sqlite_name, available
     return {
     return {
         "object_id": object_id,
         "object_id": object_id,
         "table": table,
         "table": table,
-        "columns": group_by + [alias for alias in (metric.get("alias") for metric in metrics) if alias],
+        "columns": group_by + aliases,
         "rows": result_rows,
         "rows": result_rows,
         "total": len(result_rows),
         "total": len(result_rows),
         "offset": 0,
         "offset": 0,

+ 85 - 0
tools-service/toolsvc/source.py

@@ -0,0 +1,85 @@
+"""Shared streaming input reader for uploads and processing; never guesses row repairs."""
+from __future__ import annotations
+
+import codecs
+import csv
+from pathlib import Path
+
+MAX_FIELD_SIZE = 10 * 1024 * 1024
+
+
+def detect_encoding(path: Path) -> str:
+    with path.open('rb') as handle:
+        sample = handle.read(65536)
+    if sample.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+        return 'utf-32'
+    if sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+        return 'utf-16'
+    # Validate the full stream, because a long ASCII prefix says nothing about
+    # Chinese bytes near the end. Incremental decoding handles chunk boundaries.
+    for encoding in ('utf-8-sig', 'gb18030'):
+        decoder = codecs.getincrementaldecoder(encoding)()
+        try:
+            with path.open('rb') as handle:
+                for block in iter(lambda: handle.read(1 << 20), b''):
+                    decoder.decode(block, final=False)
+                decoder.decode(b'', final=True)
+            return encoding
+        except UnicodeDecodeError:
+            continue
+    raise ValueError('无法识别文件编码,请使用 UTF-8、GB18030 或带 BOM 的 UTF-16')
+
+
+
+def iter_source(path: Path, encoding: str = 'auto', delimiter: str = 'auto'):
+    """Yield (logical data row, physical end line, headers, values); header row is 0."""
+    csv.field_size_limit(MAX_FIELD_SIZE)
+    if path.suffix.lower() in {'.xlsx', '.xlsm'}:
+        from openpyxl import load_workbook
+        workbook = load_workbook(path, read_only=True, data_only=True)
+        try:
+            headers = None
+            logical = 0
+            for line, row in enumerate(workbook.worksheets[0].iter_rows(values_only=True), 1):
+                values = ['' if cell is None else str(cell) for cell in row]
+                if headers is None:
+                    if not any(values):
+                        continue
+                    headers = values
+                    yield 0, line, headers, None
+                elif any(values):
+                    logical += 1
+                    yield logical, line, headers, values
+            if headers is None:
+                raise ValueError('输入文件为空,没有表头')
+        finally:
+            workbook.close()
+        return
+    encoding = detect_encoding(path) if encoding == 'auto' else encoding
+    with path.open(encoding=encoding, newline='') as handle:
+        if delimiter == 'auto':
+            position = handle.tell()
+            first = next((line for line in handle if line.strip()), '')
+            if not first:
+                raise ValueError('输入文件为空,没有表头')
+            # Header names contain no declaration values. Using the header avoids
+            # mistaking G_MODEL's pipes or semicolons for file delimiters.
+            delimiter = max((',', '\t', ';', '|'), key=lambda sep: len(next(csv.reader([first], delimiter=sep))))
+            handle.seek(position)
+        reader = csv.reader(handle, delimiter=delimiter, strict=True)
+        headers = None
+        logical = 0
+        try:
+            for values in reader:
+                if not values or not any(value.strip() for value in values):
+                    continue
+                if headers is None:
+                    headers = values
+                    yield 0, reader.line_num, headers, None
+                else:
+                    logical += 1
+                    yield logical, reader.line_num, headers, values
+        except csv.Error as error:
+            raise ValueError(f'CSV 第 {reader.line_num} 行读取失败:{error};请检查引号或超长字段') from error
+        if headers is None:
+            raise ValueError('输入文件为空,没有表头')

Failā izmaiņas netiks attēlotas, jo tās ir par lielu
+ 0 - 0
web/dist/assets/index-BqfzmH11.js


+ 1 - 1
web/dist/index.html

@@ -5,7 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>海关问数 Agent · 电子元器件进出口数据分析</title>
     <title>海关问数 Agent · 电子元器件进出口数据分析</title>
     <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'%3E%3Crect width='64' height='64' rx='12' fill='%231f4fd1'/%3E%3Ctext x='32' y='43' font-size='30' text-anchor='middle' fill='white' font-family='sans-serif'%3E关%3C/text%3E%3C/svg%3E" />
     <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'%3E%3Crect width='64' height='64' rx='12' fill='%231f4fd1'/%3E%3Ctext x='32' y='43' font-size='30' text-anchor='middle' fill='white' font-family='sans-serif'%3E关%3C/text%3E%3C/svg%3E" />
-    <script type="module" crossorigin src="/assets/index-Cl9HL7vR.js"></script>
+    <script type="module" crossorigin src="/assets/index-BqfzmH11.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-gwqomBW9.css">
     <link rel="stylesheet" crossorigin href="/assets/index-gwqomBW9.css">
   </head>
   </head>
   <body>
   <body>

+ 1 - 1
web/src/pages/FilesPage.tsx

@@ -187,7 +187,7 @@ export function FilesPage({ workspace, onToast, onChanged }: FilesPageProps) {
         <input
         <input
           ref={inputRef}
           ref={inputRef}
           type="file"
           type="file"
-          accept=".csv,.xlsx,.xlsm,.json,.txt"
+          accept=".csv,.tsv,.xlsx,.xlsm,.txt"
           multiple
           multiple
           hidden
           hidden
           onChange={(event) => {
           onChange={(event) => {

+ 1 - 1
web/src/pages/ProcessPage.tsx

@@ -270,7 +270,7 @@ export function ProcessPage({ workspace, onToast, onChanged }: ProcessPageProps)
             ))}
             ))}
           </select>
           </select>
           {rawFiles.length === 0 && (
           {rawFiles.length === 0 && (
-            <p className="hint warn">还没有原始数据。请到「文件」页上传海关原始 CSV,字段需包含 HS_CODE / YEAR / WEEK / MONORSUM / COUNT / G_MODEL。</p>
+            <p className="hint warn">还没有原始数据。请到「文件」页上传海关原始 CSV,支持 CSV/TSV/XLSX,中文表头与 MONORSUM_USD;自动识别常见编码和分隔符。金额须为美元。</p>
           )}
           )}
 
 
           <h3>2. 处理方式</h3>
           <h3>2. 处理方式</h3>

Daži faili netika attēloti, jo izmaiņu fails ir pārāk liels