test_legacy_fuzzy_brand.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160
  1. """80% boundary, candidate ranking, parsing and exclusion regressions."""
  2. import csv
  3. import sys
  4. import tempfile
  5. import unittest
  6. from decimal import Decimal
  7. from pathlib import Path
  8. ROOT = Path(__file__).resolve().parents[1] / 'toolsvc' / 'customs'
  9. sys.path.insert(0, str(ROOT))
  10. from toolsvc.customs.engine.brandmatch import BrandMatcher
  11. from toolsvc.customs.engine.rules import Rules
  12. from toolsvc.customs.engine.storage import ingest, REQUIRED
  13. class MatcherTests(unittest.TestCase):
  14. def matcher(self, aliases):
  15. return BrandMatcher({k: {v} for k, v in aliases.items()}, {'无品牌', '无'})
  16. def test_case_space_and_special_characters(self):
  17. m = self.matcher({'MURATA': '村田'})
  18. labels, _, method, score, _ = m.match(('m-u_r.a t a',))
  19. self.assertEqual(labels, ('村田',))
  20. self.assertEqual((method, score), ('标点归一', 100))
  21. self.assertEqual(self.matcher({'TI': '德州仪器'}).match(('t.i.',))[0], ('德州仪器',))
  22. def test_exact_eighty_percent_is_inclusive(self):
  23. m = self.matcher({'ABCDE': 'A'})
  24. self.assertEqual(m.match(('ABCDX',))[3], 80)
  25. self.assertEqual(m.match(('ABCXX',))[0], ())
  26. def test_choose_highest_similarity(self):
  27. m = self.matcher({'XBCDE': '低', 'ABCDEF': '高'})
  28. labels, _, _, score, candidates = m.match(('ABCDE',))
  29. self.assertEqual(labels, ('高',))
  30. self.assertAlmostEqual(score, 100 * 5 / 6)
  31. self.assertEqual(len(candidates), 2)
  32. def test_equal_top_scores_are_ambiguous(self):
  33. m = self.matcher({'ABCDX': 'A', 'ABCDY': 'B'})
  34. self.assertEqual(set(m.match(('ABCDE',))[0]), {'A', 'B'})
  35. def test_twenty_percent_edits_on_long_names(self):
  36. m = self.matcher({'ABCDEFGHIJ': 'A'})
  37. self.assertEqual(m.match(('ABXDEFGHIY',))[3], 80)
  38. def test_placeholders_and_short_fragments(self):
  39. m = self.matcher({'无品牌': '错误', 'TI': '错误', 'MURATA': 'M'})
  40. for raw in ('无品牌', 'T1', 'XYZ123', 'GRM188R71H104'):
  41. self.assertEqual(m.match((raw,))[0], (), raw)
  42. def test_substring_containment_below_eighty_percent(self):
  43. # “日本兴亚”只包含别名“兴亚”,覆盖率 50%,旧逻辑直接放弃。
  44. m = self.matcher({'兴亚': 'KOA'})
  45. labels, _, method, score, _ = m.match(('日本兴亚',))
  46. self.assertEqual(labels, ('KOA',))
  47. self.assertEqual(method, '子串包含')
  48. self.assertAlmostEqual(score, 50)
  49. # 关掉包含匹配后恢复旧行为。
  50. strict = BrandMatcher({'兴亚': {'KOA'}}, set(), allow_contains=False)
  51. self.assertEqual(strict.match(('日本兴亚',))[0], ())
  52. def test_containment_keeps_word_boundary_for_latin(self):
  53. m = self.matcher({'ABCD': 'A', 'YAGEO': 'Y'})
  54. # 被字母包住的片段不算命中(ABCD 在 XABCDY 里没有词边界)。
  55. self.assertEqual(m.match(('XABCDY',))[0], ())
  56. # 被中文或分隔符隔开的片段算命中。
  57. self.assertEqual(m.match(('前标YAGEO后标',))[0], ('Y',))
  58. # 拉丁名短于 4 个字符不做低覆盖率包含,避免缩写误命中。
  59. self.assertEqual(self.matcher({'KOA': 'K'}).match(('XKOAX',))[0], ())
  60. class IntegrationTests(unittest.TestCase):
  61. @classmethod
  62. def setUpClass(cls):
  63. cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx',
  64. ROOT / 'rules/进出口数据字段需求20260819.xlsx')
  65. def test_fuzzy_in_real_field_formats(self):
  66. for raw in ('品牌:MURTA;型号:TEST-1',
  67. 'MURTA/TEST-1/计算机用',
  68. '4|3|片式|瓷介|MURTA|TEST-1'):
  69. p = self.rules.parse('8532241000', raw)
  70. self.assertEqual((p['brand'], p['model']), ('MURATA', 'TEST-1'), raw)
  71. self.assertAlmostEqual(p['brand_match_score'], 83.333333)
  72. self.assertEqual(p['brand_match_method'], '拼写近似')
  73. def test_labelled_and_standalone_punctuated_brand(self):
  74. for raw in ('品牌:m-u_r.a t a;型号:TEST-1', 'm-u_r.a t a'):
  75. p = self.rules.parse('8532241000', raw)
  76. self.assertEqual(p['brand'], 'MURATA', raw)
  77. self.assertEqual(p['brand_match_score'], 100)
  78. def test_model_or_usage_label_is_not_a_brand(self):
  79. for raw in ('型号:MURTA', '用途:MURTA'):
  80. self.assertEqual(self.rules.parse('8532241000', raw)['brand'], '')
  81. def test_alias_fragment_recovers_brand_inside_longer_text(self):
  82. # 海关写法“品牌:日本兴亚”,规则表里只有“KOA/兴亚”“KOA,日本兴亚”等整串。
  83. for raw in ('品牌:日本兴亚', '日本兴亚', '品:KOA等 中文名称:兴亚等'):
  84. p = self.rules.parse('8532241000', raw)
  85. self.assertEqual(p['brand'], 'KOA', raw)
  86. p = self.rules.parse('8532241000', '品牌:日本兴亚电子')
  87. self.assertEqual(p['brand'], 'KOA')
  88. self.assertEqual(p['brand_match_method'], '子串包含')
  89. self.assertIn('品牌包含匹配', p['flags'])
  90. def test_model_containment_recovers_embedded_model(self):
  91. # 已知型号被包在更长的申报文本里,旧逻辑整条判为未识别。
  92. for raw in ('型号:全新原装K4RAH086VE-E0000EL',
  93. '品牌:SAMSUNG;型号:贴片K4RAH086VE-E0000EL'):
  94. p = self.rules.parse('8542321010', raw)
  95. self.assertEqual(p['model'], 'K4RAH086VE-E0000EL', raw)
  96. self.assertIn('型号包含匹配', p['flags'])
  97. self.assertEqual(p['c2'], 'DRAM', raw)
  98. # 括号包裹的值现在先被清理成干净型号、直接精确命中,不再需要包含匹配兜底
  99. p = self.rules.parse('8542321010', '型号:(K4RAH086VE-E0000EL)')
  100. self.assertEqual(p['model'], 'K4RAH086VE-E0000EL')
  101. self.assertEqual(p['c2'], 'DRAM')
  102. def test_model_containment_only_for_memory_hs(self):
  103. # 型号表只覆盖存储器 4 个 HS,其他 HS 不能凭猜测套用。
  104. p = self.rules.parse('8532241000', '型号:全新原装K4RAH086VE-E0000EL')
  105. self.assertNotEqual(p['model'], 'K4RAH086VE-E0000EL')
  106. self.assertEqual(p['mapped'], 0)
  107. def test_containment_can_be_switched_off(self):
  108. strict = Rules(ROOT / 'rules/数据映射规则.xlsx',
  109. ROOT / 'rules/进出口数据字段需求20260819.xlsx',
  110. fuzzy_contains=False)
  111. self.assertEqual(strict.parse('8532241000', '品牌:日本兴亚电子')['brand'], '')
  112. self.assertNotEqual(strict.parse('8542321010', '型号:全新原装K4RAH086VE-E0000EL')['model'],
  113. 'K4RAH086VE-E0000EL')
  114. def test_empty_gmodel_excluded_from_all_statistics(self):
  115. with tempfile.TemporaryDirectory() as folder:
  116. out = Path(folder)
  117. source = out / 'input.csv'
  118. with source.open('w', encoding='utf-8-sig', newline='') as f:
  119. writer = csv.writer(f)
  120. writer.writerow(REQUIRED)
  121. for value in ('', ' \t\u3000', '品牌:MURTA;型号:TEST-1'):
  122. writer.writerow(['I', '8532241000', '2025', '1', '100', '10', value])
  123. config = dict(input_csv=str(source), encoding='utf-8-sig', delimiter=',',
  124. cutoff_date='2026-08-31', directions=['I'], start_year=2025, end_year=2026)
  125. db, stats = ingest(config, self.rules, out)
  126. try:
  127. self.assertEqual((stats['total'], stats['included'], stats['excluded']), (3, 1, 2))
  128. self.assertEqual(db.execute('SELECT count(*) FROM audit').fetchone()[0], 1)
  129. for table in ('fact', 'monthly'):
  130. rows = db.execute(f'SELECT amount, quantity FROM {table}').fetchall()
  131. self.assertEqual(sum(Decimal(r[0]) for r in rows), 100)
  132. self.assertEqual(sum(Decimal(r[1]) for r in rows), 10)
  133. self.assertEqual(stats['exclusion_reasons']['G_MODEL为空'], 2)
  134. finally:
  135. db.close()
  136. if __name__ == '__main__':
  137. unittest.main()