| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160 |
- """80% boundary, candidate ranking, parsing and exclusion regressions."""
- import csv
- import sys
- import tempfile
- import unittest
- from decimal import Decimal
- from pathlib import Path
- ROOT = Path(__file__).resolve().parents[1] / 'toolsvc' / 'customs'
- sys.path.insert(0, str(ROOT))
- from toolsvc.customs.engine.brandmatch import BrandMatcher
- from toolsvc.customs.engine.rules import Rules
- from toolsvc.customs.engine.storage import ingest, REQUIRED
- class MatcherTests(unittest.TestCase):
- def matcher(self, aliases):
- return BrandMatcher({k: {v} for k, v in aliases.items()}, {'无品牌', '无'})
- def test_case_space_and_special_characters(self):
- m = self.matcher({'MURATA': '村田'})
- labels, _, method, score, _ = m.match(('m-u_r.a t a',))
- self.assertEqual(labels, ('村田',))
- self.assertEqual((method, score), ('标点归一', 100))
- self.assertEqual(self.matcher({'TI': '德州仪器'}).match(('t.i.',))[0], ('德州仪器',))
- def test_exact_eighty_percent_is_inclusive(self):
- m = self.matcher({'ABCDE': 'A'})
- self.assertEqual(m.match(('ABCDX',))[3], 80)
- self.assertEqual(m.match(('ABCXX',))[0], ())
- def test_choose_highest_similarity(self):
- m = self.matcher({'XBCDE': '低', 'ABCDEF': '高'})
- labels, _, _, score, candidates = m.match(('ABCDE',))
- self.assertEqual(labels, ('高',))
- self.assertAlmostEqual(score, 100 * 5 / 6)
- self.assertEqual(len(candidates), 2)
- def test_equal_top_scores_are_ambiguous(self):
- m = self.matcher({'ABCDX': 'A', 'ABCDY': 'B'})
- self.assertEqual(set(m.match(('ABCDE',))[0]), {'A', 'B'})
- def test_twenty_percent_edits_on_long_names(self):
- m = self.matcher({'ABCDEFGHIJ': 'A'})
- self.assertEqual(m.match(('ABXDEFGHIY',))[3], 80)
- def test_placeholders_and_short_fragments(self):
- m = self.matcher({'无品牌': '错误', 'TI': '错误', 'MURATA': 'M'})
- for raw in ('无品牌', 'T1', 'XYZ123', 'GRM188R71H104'):
- self.assertEqual(m.match((raw,))[0], (), raw)
- def test_substring_containment_below_eighty_percent(self):
- # “日本兴亚”只包含别名“兴亚”,覆盖率 50%,旧逻辑直接放弃。
- m = self.matcher({'兴亚': 'KOA'})
- labels, _, method, score, _ = m.match(('日本兴亚',))
- self.assertEqual(labels, ('KOA',))
- self.assertEqual(method, '子串包含')
- self.assertAlmostEqual(score, 50)
- # 关掉包含匹配后恢复旧行为。
- strict = BrandMatcher({'兴亚': {'KOA'}}, set(), allow_contains=False)
- self.assertEqual(strict.match(('日本兴亚',))[0], ())
- def test_containment_keeps_word_boundary_for_latin(self):
- m = self.matcher({'ABCD': 'A', 'YAGEO': 'Y'})
- # 被字母包住的片段不算命中(ABCD 在 XABCDY 里没有词边界)。
- self.assertEqual(m.match(('XABCDY',))[0], ())
- # 被中文或分隔符隔开的片段算命中。
- self.assertEqual(m.match(('前标YAGEO后标',))[0], ('Y',))
- # 拉丁名短于 4 个字符不做低覆盖率包含,避免缩写误命中。
- self.assertEqual(self.matcher({'KOA': 'K'}).match(('XKOAX',))[0], ())
- class IntegrationTests(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx',
- ROOT / 'rules/进出口数据字段需求20260819.xlsx')
- def test_fuzzy_in_real_field_formats(self):
- for raw in ('品牌:MURTA;型号:TEST-1',
- 'MURTA/TEST-1/计算机用',
- '4|3|片式|瓷介|MURTA|TEST-1'):
- p = self.rules.parse('8532241000', raw)
- self.assertEqual((p['brand'], p['model']), ('MURATA', 'TEST-1'), raw)
- self.assertAlmostEqual(p['brand_match_score'], 83.333333)
- self.assertEqual(p['brand_match_method'], '拼写近似')
- def test_labelled_and_standalone_punctuated_brand(self):
- for raw in ('品牌:m-u_r.a t a;型号:TEST-1', 'm-u_r.a t a'):
- p = self.rules.parse('8532241000', raw)
- self.assertEqual(p['brand'], 'MURATA', raw)
- self.assertEqual(p['brand_match_score'], 100)
- def test_model_or_usage_label_is_not_a_brand(self):
- for raw in ('型号:MURTA', '用途:MURTA'):
- self.assertEqual(self.rules.parse('8532241000', raw)['brand'], '')
- def test_alias_fragment_recovers_brand_inside_longer_text(self):
- # 海关写法“品牌:日本兴亚”,规则表里只有“KOA/兴亚”“KOA,日本兴亚”等整串。
- for raw in ('品牌:日本兴亚', '日本兴亚', '品:KOA等 中文名称:兴亚等'):
- p = self.rules.parse('8532241000', raw)
- self.assertEqual(p['brand'], 'KOA', raw)
- p = self.rules.parse('8532241000', '品牌:日本兴亚电子')
- self.assertEqual(p['brand'], 'KOA')
- self.assertEqual(p['brand_match_method'], '子串包含')
- self.assertIn('品牌包含匹配', p['flags'])
- def test_model_containment_recovers_embedded_model(self):
- # 已知型号被包在更长的申报文本里,旧逻辑整条判为未识别。
- for raw in ('型号:全新原装K4RAH086VE-E0000EL',
- '品牌:SAMSUNG;型号:贴片K4RAH086VE-E0000EL'):
- p = self.rules.parse('8542321010', raw)
- self.assertEqual(p['model'], 'K4RAH086VE-E0000EL', raw)
- self.assertIn('型号包含匹配', p['flags'])
- self.assertEqual(p['c2'], 'DRAM', raw)
- # 括号包裹的值现在先被清理成干净型号、直接精确命中,不再需要包含匹配兜底
- p = self.rules.parse('8542321010', '型号:(K4RAH086VE-E0000EL)')
- self.assertEqual(p['model'], 'K4RAH086VE-E0000EL')
- self.assertEqual(p['c2'], 'DRAM')
- def test_model_containment_only_for_memory_hs(self):
- # 型号表只覆盖存储器 4 个 HS,其他 HS 不能凭猜测套用。
- p = self.rules.parse('8532241000', '型号:全新原装K4RAH086VE-E0000EL')
- self.assertNotEqual(p['model'], 'K4RAH086VE-E0000EL')
- self.assertEqual(p['mapped'], 0)
- def test_containment_can_be_switched_off(self):
- strict = Rules(ROOT / 'rules/数据映射规则.xlsx',
- ROOT / 'rules/进出口数据字段需求20260819.xlsx',
- fuzzy_contains=False)
- self.assertEqual(strict.parse('8532241000', '品牌:日本兴亚电子')['brand'], '')
- self.assertNotEqual(strict.parse('8542321010', '型号:全新原装K4RAH086VE-E0000EL')['model'],
- 'K4RAH086VE-E0000EL')
- def test_empty_gmodel_excluded_from_all_statistics(self):
- with tempfile.TemporaryDirectory() as folder:
- out = Path(folder)
- source = out / 'input.csv'
- with source.open('w', encoding='utf-8-sig', newline='') as f:
- writer = csv.writer(f)
- writer.writerow(REQUIRED)
- for value in ('', ' \t\u3000', '品牌:MURTA;型号:TEST-1'):
- writer.writerow(['I', '8532241000', '2025', '1', '100', '10', value])
- config = dict(input_csv=str(source), encoding='utf-8-sig', delimiter=',',
- cutoff_date='2026-08-31', directions=['I'], start_year=2025, end_year=2026)
- db, stats = ingest(config, self.rules, out)
- try:
- self.assertEqual((stats['total'], stats['included'], stats['excluded']), (3, 1, 2))
- self.assertEqual(db.execute('SELECT count(*) FROM audit').fetchone()[0], 1)
- for table in ('fact', 'monthly'):
- rows = db.execute(f'SELECT amount, quantity FROM {table}').fetchall()
- self.assertEqual(sum(Decimal(r[0]) for r in rows), 100)
- self.assertEqual(sum(Decimal(r[1]) for r in rows), 10)
- self.assertEqual(stats['exclusion_reasons']['G_MODEL为空'], 2)
- finally:
- db.close()
- if __name__ == '__main__':
- unittest.main()
|