"""80% boundary, candidate ranking, parsing and exclusion regressions.""" import csv import sys import tempfile import unittest from decimal import Decimal from pathlib import Path ROOT = Path(__file__).resolve().parents[1] / 'toolsvc' / 'customs' sys.path.insert(0, str(ROOT)) from toolsvc.customs.engine.brandmatch import BrandMatcher from toolsvc.customs.engine.rules import Rules from toolsvc.customs.engine.storage import ingest, REQUIRED class MatcherTests(unittest.TestCase): def matcher(self, aliases): return BrandMatcher({k: {v} for k, v in aliases.items()}, {'无品牌', '无'}) def test_case_space_and_special_characters(self): m = self.matcher({'MURATA': '村田'}) labels, _, method, score, _ = m.match(('m-u_r.a t a',)) self.assertEqual(labels, ('村田',)) self.assertEqual((method, score), ('标点归一', 100)) self.assertEqual(self.matcher({'TI': '德州仪器'}).match(('t.i.',))[0], ('德州仪器',)) def test_exact_eighty_percent_is_inclusive(self): m = self.matcher({'ABCDE': 'A'}) self.assertEqual(m.match(('ABCDX',))[3], 80) self.assertEqual(m.match(('ABCXX',))[0], ()) def test_choose_highest_similarity(self): m = self.matcher({'XBCDE': '低', 'ABCDEF': '高'}) labels, _, _, score, candidates = m.match(('ABCDE',)) self.assertEqual(labels, ('高',)) self.assertAlmostEqual(score, 100 * 5 / 6) self.assertEqual(len(candidates), 2) def test_equal_top_scores_are_ambiguous(self): m = self.matcher({'ABCDX': 'A', 'ABCDY': 'B'}) self.assertEqual(set(m.match(('ABCDE',))[0]), {'A', 'B'}) def test_twenty_percent_edits_on_long_names(self): m = self.matcher({'ABCDEFGHIJ': 'A'}) self.assertEqual(m.match(('ABXDEFGHIY',))[3], 80) def test_placeholders_and_short_fragments(self): m = self.matcher({'无品牌': '错误', 'TI': '错误', 'MURATA': 'M'}) for raw in ('无品牌', 'T1', 'XYZ123', 'GRM188R71H104'): self.assertEqual(m.match((raw,))[0], (), raw) def test_substring_containment_below_eighty_percent(self): # “日本兴亚”只包含别名“兴亚”,覆盖率 50%,旧逻辑直接放弃。 m = self.matcher({'兴亚': 'KOA'}) labels, _, method, score, _ = m.match(('日本兴亚',)) self.assertEqual(labels, ('KOA',)) self.assertEqual(method, '子串包含') self.assertAlmostEqual(score, 50) # 关掉包含匹配后恢复旧行为。 strict = BrandMatcher({'兴亚': {'KOA'}}, set(), allow_contains=False) self.assertEqual(strict.match(('日本兴亚',))[0], ()) def test_containment_keeps_word_boundary_for_latin(self): m = self.matcher({'ABCD': 'A', 'YAGEO': 'Y'}) # 被字母包住的片段不算命中(ABCD 在 XABCDY 里没有词边界)。 self.assertEqual(m.match(('XABCDY',))[0], ()) # 被中文或分隔符隔开的片段算命中。 self.assertEqual(m.match(('前标YAGEO后标',))[0], ('Y',)) # 拉丁名短于 4 个字符不做低覆盖率包含,避免缩写误命中。 self.assertEqual(self.matcher({'KOA': 'K'}).match(('XKOAX',))[0], ()) class IntegrationTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.rules = Rules(ROOT / 'rules/数据映射规则.xlsx', ROOT / 'rules/进出口数据字段需求20260819.xlsx') def test_fuzzy_in_real_field_formats(self): for raw in ('品牌:MURTA;型号:TEST-1', 'MURTA/TEST-1/计算机用', '4|3|片式|瓷介|MURTA|TEST-1'): p = self.rules.parse('8532241000', raw) self.assertEqual((p['brand'], p['model']), ('MURATA', 'TEST-1'), raw) self.assertAlmostEqual(p['brand_match_score'], 83.333333) self.assertEqual(p['brand_match_method'], '拼写近似') def test_labelled_and_standalone_punctuated_brand(self): for raw in ('品牌:m-u_r.a t a;型号:TEST-1', 'm-u_r.a t a'): p = self.rules.parse('8532241000', raw) self.assertEqual(p['brand'], 'MURATA', raw) self.assertEqual(p['brand_match_score'], 100) def test_model_or_usage_label_is_not_a_brand(self): for raw in ('型号:MURTA', '用途:MURTA'): self.assertEqual(self.rules.parse('8532241000', raw)['brand'], '') def test_alias_fragment_recovers_brand_inside_longer_text(self): # 海关写法“品牌:日本兴亚”,规则表里只有“KOA/兴亚”“KOA,日本兴亚”等整串。 for raw in ('品牌:日本兴亚', '日本兴亚', '品:KOA等 中文名称:兴亚等'): p = self.rules.parse('8532241000', raw) self.assertEqual(p['brand'], 'KOA', raw) p = self.rules.parse('8532241000', '品牌:日本兴亚电子') self.assertEqual(p['brand'], 'KOA') self.assertEqual(p['brand_match_method'], '子串包含') self.assertIn('品牌包含匹配', p['flags']) def test_model_containment_recovers_embedded_model(self): # 已知型号被包在更长的申报文本里,旧逻辑整条判为未识别。 for raw in ('型号:全新原装K4RAH086VE-E0000EL', '品牌:SAMSUNG;型号:贴片K4RAH086VE-E0000EL'): p = self.rules.parse('8542321010', raw) self.assertEqual(p['model'], 'K4RAH086VE-E0000EL', raw) self.assertIn('型号包含匹配', p['flags']) self.assertEqual(p['c2'], 'DRAM', raw) # 括号包裹的值现在先被清理成干净型号、直接精确命中,不再需要包含匹配兜底 p = self.rules.parse('8542321010', '型号:(K4RAH086VE-E0000EL)') self.assertEqual(p['model'], 'K4RAH086VE-E0000EL') self.assertEqual(p['c2'], 'DRAM') def test_model_containment_only_for_memory_hs(self): # 型号表只覆盖存储器 4 个 HS,其他 HS 不能凭猜测套用。 p = self.rules.parse('8532241000', '型号:全新原装K4RAH086VE-E0000EL') self.assertNotEqual(p['model'], 'K4RAH086VE-E0000EL') self.assertEqual(p['mapped'], 0) def test_containment_can_be_switched_off(self): strict = Rules(ROOT / 'rules/数据映射规则.xlsx', ROOT / 'rules/进出口数据字段需求20260819.xlsx', fuzzy_contains=False) self.assertEqual(strict.parse('8532241000', '品牌:日本兴亚电子')['brand'], '') self.assertNotEqual(strict.parse('8542321010', '型号:全新原装K4RAH086VE-E0000EL')['model'], 'K4RAH086VE-E0000EL') def test_empty_gmodel_excluded_from_all_statistics(self): with tempfile.TemporaryDirectory() as folder: out = Path(folder) source = out / 'input.csv' with source.open('w', encoding='utf-8-sig', newline='') as f: writer = csv.writer(f) writer.writerow(REQUIRED) for value in ('', ' \t\u3000', '品牌:MURTA;型号:TEST-1'): writer.writerow(['I', '8532241000', '2025', '1', '100', '10', value]) config = dict(input_csv=str(source), encoding='utf-8-sig', delimiter=',', cutoff_date='2026-08-31', directions=['I'], start_year=2025, end_year=2026) db, stats = ingest(config, self.rules, out) try: self.assertEqual((stats['total'], stats['included'], stats['excluded']), (3, 1, 2)) self.assertEqual(db.execute('SELECT count(*) FROM audit').fetchone()[0], 1) for table in ('fact', 'monthly'): rows = db.execute(f'SELECT amount, quantity FROM {table}').fetchall() self.assertEqual(sum(Decimal(r[0]) for r in rows), 100) self.assertEqual(sum(Decimal(r[1]) for r in rows), 10) self.assertEqual(stats['exclusion_reasons']['G_MODEL为空'], 2) finally: db.close() if __name__ == '__main__': unittest.main()