rules.py 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849
  1. """Workbook rules, two-stage application matching and conservative G_MODEL parsing."""
  2. import re
  3. import unicodedata
  4. from collections import Counter, defaultdict
  5. from functools import lru_cache
  6. from openpyxl import load_workbook
  7. from .structure import (BRAND_QUALIFIER, CHINESE_STATUS, KEY_TOKEN, NO_BRAND, NO_MODEL,
  8. clean_value, label_kind, labels_in, looks_like_model,
  9. split_attributes, strip_brand_suffix, strip_model_suffix)
  10. from .brandmatch import BrandMatcher
  11. MEMORY_HS = {'8542321010', '8542321090', '8542329010', '8542329090'}
  12. # 这些词在品牌位上只表示“没有/不适用”,不能当成真实品牌参与自动扫描。
  13. PLACEHOLDER = {'无', '无品牌', '无牌', '不适用', '其他', '其它', 'NA', 'N/A', 'NONE', 'NIL',
  14. '待定', '未提供', '未知', '未识别', '-', '—', '/'}
  15. # 别名字符串里常见的说明性前缀,拆片段时要先去掉。
  16. ALIAS_LABEL = re.compile(
  17. r'(?:中文名称|外文名称|中文名|外文名|英文名称|品牌名称|商品名称|品牌|品名|品|BRAND|MODEL|NAME)'
  18. r'\s*[::=]\s*', re.I)
  19. ALIAS_SPLIT = re.compile(r'[,,、//;;||\s]+')
  20. # 拆出来的片段若只是通用词或国别,作为品牌线索只会制造误命中,不建索引。
  21. ALIAS_STOP = {
  22. '英文', '中文', '外文', '品牌', '品名', '名称', '商品名称', '型号', '等',
  23. '无中文名称', '无外文名称', '无中文品牌', '无外文品牌', '无中文名', '无外文名',
  24. '无品牌', '无牌', '其他', '其它', '未知', '不适用', '不详', '不祥',
  25. '株式会社', '有限公司', '公司', '集团', '电子', '电气', '半导体', '科技', '芯片',
  26. '中国', '日本', '韩国', '美国', '德国', '法国', '英国', '荷兰', '瑞士', '奥地利',
  27. '新加坡', '马来西亚', '泰国', '越南', '印度', '台湾', '中国台湾', '香港', '深圳',
  28. }
  29. def alias_tokens(raw):
  30. """把一个申报写法拆成可索引的片段:整串、去标签、括号内外、按分隔符切开的各项。
  31. 真实申报里经常写成“品:KOA等 中文名称:兴亚等”或“KOA/兴亚”“KOA,日本兴亚”,
  32. 只按整串建索引时,“品牌:日本兴亚”这类写法永远匹配不上;拆出片段后即可命中。
  33. """
  34. text = unicodedata.normalize('NFKC', str(raw or '')).strip()
  35. out = []
  36. def add(value):
  37. value = str(value or '').strip().strip('"\'“”‘’ \t,,;;/.')
  38. if value.endswith('等') and len(value) > 1:
  39. value = value[:-1]
  40. if value and value not in out:
  41. out.append(value)
  42. add(text)
  43. body = ALIAS_LABEL.sub(' ', text)
  44. add(body)
  45. for inner in re.findall(r'[((]([^))]*)[))]', body):
  46. add(inner)
  47. for part in ALIAS_SPLIT.split(re.sub(r'[((][^))]*[))]', ' ', body)):
  48. add(part)
  49. return out
  50. def norm(value):
  51. return re.sub(r'\s+', '', unicodedata.normalize('NFKC', str(value or ''))).upper()
  52. # 纯电气参数:容值、耐压、功率、尺寸、频率,这些不是型号。
  53. SPEC_LIKE = re.compile(
  54. r'^(?:\d+(?:\.\d+)?\s*(?:[nuµμpP]?F|[VvWwAa]|Hz|KHz|MHz|%|Ω|ohm)|\d{4}|\d+(?:\.\d+)?)$')
  55. # 像品牌名的写法(纯拉丁,不含中文),用来提示“映射表里可能缺这个品牌”。
  56. LATIN_ONLY = re.compile(r'[A-Za-z][A-Za-z0-9 .&+\-/]{1,30}')
  57. # 规则表里的兜底标签:不是真品牌,模糊/拼写/包含命中到它们一律不认。
  58. CATCH_ALL = {'Others', '其他', '其它', '其他品牌', '未识别', '未知', 'UNKNOWN'}
  59. # 明显是“品牌字段”的写法(带 牌/品牌 字样),用于在品牌认不出时仍能取到后面的型号。
  60. BRAND_FIELD = re.compile(r'(?:牌|品牌|商标)\s*$')
  61. def model_candidate(text, is_brand=None, app_signal=None):
  62. """判断一段文字能否当型号用:不是参数、不是别的标签的值、有字母也有数字。"""
  63. value = clean_value(text)
  64. if not value or SPEC_LIKE.match(value):
  65. return ''
  66. labels = list(labels_in(value))
  67. if labels and not any(kind in ('品牌', 'BRAND', '型号', 'MODEL') for kind, _ in labels):
  68. return ''
  69. if not (re.search(r'[A-Za-z]', value) and re.search(r'\d', value)):
  70. return ''
  71. if is_brand and is_brand(value):
  72. return ''
  73. if app_signal and app_signal(value):
  74. return ''
  75. return value
  76. def canon_label(value):
  77. """品牌标签的归并键:去掉括号说明与空白后大写。"""
  78. text = unicodedata.normalize('NFKC', str(value or '')).strip()
  79. text = re.sub(r'[((][^))]*[))]', '', text).strip()
  80. return re.sub(r'\s+', '', text).upper()
  81. def unique(values):
  82. values = set(values)
  83. return next(iter(values)) if len(values) == 1 else ''
  84. class Rules:
  85. def __init__(self, mapping, fields, country_system='ISO', fuzzy_contains=True):
  86. self.alias, self.brand_country, self.models = defaultdict(set), defaultdict(set), defaultdict(set)
  87. self.prefixes, self.first, self.second, self.cap, self.schema = [], {}, [], {}, {}
  88. self.conflicts = []
  89. self.fuzzy_contains = fuzzy_contains
  90. w = load_workbook(mapping, read_only=True, data_only=True)
  91. pairs = [(str(b).strip(), str(r).strip())
  92. for b, r in w['品牌名称映射规则'].iter_rows(min_row=2, values_only=True) if b and r]
  93. # 品牌标签存在 “MICRON/Micron”“AMD/AMD(XILINX)” 等同一品牌的多种写法,
  94. # 会让同一申报内容映射到多个标签而被整条判为冲突。先归并再建别名表。
  95. counts = Counter(label for label, _ in pairs)
  96. display = {}
  97. for label in counts:
  98. key = canon_label(label)
  99. current = display.get(key)
  100. if current is None or ((counts[label], -len(label), label)
  101. > (counts[current], -len(current), current)):
  102. display[key] = label
  103. self.label_map = {label: display[canon_label(label)] for label in counts}
  104. self.label_merges = sorted((a, b) for a, b in self.label_map.items() if a != b)
  105. full = defaultdict(set)
  106. for brand, raw in pairs:
  107. shown = self.label_map[brand]
  108. full[norm(raw)].add(shown)
  109. full[norm(brand)].add(shown)
  110. # 别名字符串常带说明前缀或并列写法。整串之外再按片段建索引,
  111. # “品牌:日本兴亚”这类只含品牌一部分的写法才能命中兴亚(KOA)。
  112. extra = defaultdict(set)
  113. for brand, raw in pairs:
  114. shown = self.label_map[brand]
  115. for token in alias_tokens(raw):
  116. key = norm(token)
  117. if not key or key in full or key in ALIAS_STOP:
  118. continue
  119. if len(key) < (2 if re.search(r'[\u3400-\u9fff]', key) else 3):
  120. continue
  121. extra[key].add(shown)
  122. for key, labels in extra.items():
  123. # 片段若指向多个品牌(说明是“电子”“英文”这类通用词),保留只会制造冲突。
  124. if len(labels) == 1:
  125. full[key] = labels
  126. self.alias = full
  127. # 记录“由片段推断出来”的键:型号表里的品牌中文名是显式品牌名,可以覆盖这类推断键。
  128. self.fragment_keys = {key for key, labels in extra.items() if len(labels) == 1}
  129. for brand, country in w['品牌国别映射规则'].iter_rows(min_row=2, values_only=True):
  130. if brand and country:
  131. label = str(brand).strip()
  132. self.brand_country[norm(self.label_map.get(label, label))].add(str(country).strip())
  133. for r in w['存储器型号分类映射规则'].iter_rows(min_row=2, values_only=True):
  134. en, cn, model, prefix, desc, c1, c2, c3, c4 = r
  135. if not model:
  136. continue
  137. brand = unique(self.alias[norm(en)]) or str(en or cn or '')
  138. for alias in (en, cn):
  139. if alias:
  140. # 型号表只补空位,或覆盖“由别名片段推断出来”的键(如 超微→SUPERMICRO);
  141. # 名称表里作为整串别名出现过的键不改写,避免同一写法被两个品牌认领。
  142. key = norm(alias)
  143. current = self.alias.get(key)
  144. if key in self.fragment_keys:
  145. self.alias[key] = {brand}
  146. elif not current or brand in current:
  147. self.alias[key].add(brand)
  148. record = (brand, str(desc or ''), str(c2 or ''), str(c3 or ''), str(c4 or ''),str(c1 or ''))
  149. self.models[norm(model)].add(record)
  150. if prefix:
  151. self.prefixes.append((norm(prefix), record))
  152. self.prefixes = sorted(set(self.prefixes), key=lambda x: -len(x[0]))
  153. self.prefix_lookup=defaultdict(set)
  154. for prefix,record in self.prefixes:self.prefix_lookup[prefix].add(record)
  155. # 四字片段索引:从任意一段文本里反查“里面藏着哪个已知型号”,
  156. # 用于“型号:K4B4G1646E等”“K4B4G1646E-BYK0”这类写法,避免全表逐条扫描。
  157. self.model_grams = defaultdict(set)
  158. for key in self.models:
  159. for i in range(len(key) - 3):
  160. self.model_grams[key[i:i + 4]].add(key)
  161. for r in w['应用领域映射规则(一轮匹配)'].iter_rows(min_row=2, values_only=True):
  162. if r[0] and r[1]:
  163. self.first.setdefault(norm(r[0]), set()).add(str(r[1]))
  164. for r in w['应用领域与功能映射规则(二轮匹配)'].iter_rows(min_row=2, values_only=True):
  165. if r[0] not in ('其他', '通用领域') and r[1]:
  166. self.second.append((r[0], [norm(k) for k in str(r[1]).split('、')]))
  167. for r in w['电容器映射规则'].iter_rows(min_row=2, values_only=True):
  168. if r[2]:
  169. self.cap[str(r[2])[:10]] = str(r[1])
  170. w.close()
  171. for kind, mapping in [('品牌', self.alias), ('品牌国别', self.brand_country), ('型号', self.models), ('应用一轮', self.first)]:
  172. for key, values in mapping.items():
  173. if len(values) > 1:
  174. self.conflicts.append((kind, key, repr(sorted(values))))
  175. for original, merged in self.label_merges:
  176. self.conflicts.append(('品牌标签归并', original, f'归并为 {merged}'))
  177. self.brand_matcher = BrandMatcher(self.alias, PLACEHOLDER | {
  178. '无中文品牌', '无外文品牌', '无中文名称', '无外文名称'}, allow_contains=fuzzy_contains)
  179. self._build_reverse_index()
  180. w = load_workbook(fields, read_only=True, data_only=True)
  181. for r in w['字段规则_申报要素'].iter_rows(min_row=3, values_only=True):
  182. if r[0]:
  183. self.schema[str(r[0])] = [str(x or '') for x in r[2:]]
  184. country_candidates=defaultdict(set)
  185. self.countries, self.ports = {}, {}
  186. for r in w['映射规则_国别地区代码'].iter_rows(min_row=2, values_only=True):
  187. if not r[6]:
  188. continue
  189. keys = [r[0]] if country_system == 'GEC' else [r[1], r[2], r[3]]
  190. for key in keys + [r[6]]:
  191. if key is not None:
  192. if str(key) in ['不适用','无']:continue
  193. country_candidates[norm(key)].add(str(r[6]))
  194. if str(key).isdigit():
  195. country_candidates[str(key).zfill(3)].add(str(r[6]))
  196. for key,values in country_candidates.items():
  197. if len(values)==1:self.countries[key]=next(iter(values))
  198. else:
  199. self.countries[key]='映射冲突:'+key
  200. self.conflicts.append(('国家地区',key,repr(sorted(values))))
  201. for code, cn, en, country in w['映射规则_港口代码'].iter_rows(min_row=2, values_only=True):
  202. if code:
  203. self.ports[norm(code)] = (str(cn or en or code), str(country or '未识别'))
  204. w.close()
  205. def country(self, raw):
  206. return self.countries.get(norm(raw), f'未映射:{raw}' if raw else '未识别')
  207. def port(self, raw):
  208. key = norm(raw)
  209. return self.ports.get(key, (f'未映射:{raw}', '未识别')) if key else ('未识别', '未识别')
  210. # ---------- 品牌 / 型号取词 ----------
  211. @staticmethod
  212. def _clean(raw):
  213. text = unicodedata.normalize('NFKC', str(raw or '')).strip()
  214. return text.strip('"\'“”‘’ \t').strip()
  215. def brand_variants(self, raw):
  216. """按可靠性从高到低给出品牌候选写法:整体、去“牌”、括号内外、枚举各项。"""
  217. text = self._clean(raw)
  218. text = re.sub(r'^(?:品牌|BRAND)\s*[::=]\s*', '', text, flags=re.I)
  219. out = []
  220. def add(value):
  221. value = str(value or '').strip().strip('"\'“”‘’ \t').strip()
  222. if value and value not in out:
  223. out.append(value)
  224. add(text)
  225. if text.endswith('牌') and len(text) > 1:
  226. add(text[:-1])
  227. match = re.match(r'^(.*?)[((]([^))]*)[))](.*)$', text)
  228. if match:
  229. add(match.group(1) + match.group(3))
  230. add(match.group(1))
  231. add(match.group(2))
  232. if match.group(2).endswith('牌') and len(match.group(2)) > 1:
  233. add(match.group(2)[:-1])
  234. flat = re.sub(r'[((][^))]*[))]', ' ', text)
  235. for part in re.split(r'[,,、//;;||]+', flat):
  236. add(part)
  237. for part in re.split(r'[\s_]+', text):
  238. add(part)
  239. return out
  240. def _build_reverse_index(self):
  241. """为"规则表里的长申报内容包含 G_MODEL 写法"建索引。
  242. 品牌表里有很多长串,比如 `PANJIT牌/强茂牌`、`村田制作所(MURATA)`、
  243. `NEXPERIA、DIODES、PANJIT/无中文名称`。反过来查"这段文字出现在哪条规则里",
  244. 用 3 字片倒排 + 逐条验证,避免每次全表扫描。
  245. """
  246. self._rev_keys = [] # [(规范化的键, 该键对应的品牌集合)]
  247. self._rev_grams = defaultdict(set) # 二字片 -> 键序号集合(中文两字就是完整品牌)
  248. for key, labels in self.alias.items():
  249. packed = norm(key)
  250. if len(packed) < 2:
  251. continue
  252. index = len(self._rev_keys)
  253. self._rev_keys.append((packed, frozenset(labels)))
  254. for gram in {packed[i:i + 2] for i in range(len(packed) - 1)}:
  255. self._rev_grams[gram].add(index)
  256. @lru_cache(maxsize=16384)
  257. def reverse_brand_labels(self, text):
  258. """返回"规则表里的哪条申报内容包含这段文字"所对应的品牌。
  259. 例:'PANJIT' 会被 `PANJIT牌/强茂牌` 命中(长申报内容里含英文名)。
  260. 只认"除这段文字外没有别的已知品牌"的那几条规则——像
  261. `NEXPERIA、DIODES、PANJIT/无中文名称` 这种多品牌混写不算,
  262. 否则会把 PANJIT 认成 Nexperia。
  263. """
  264. packed = norm(text)
  265. # 中文两字就可能是完整品牌(村田/强茂),拉丁字母要四个以上才不至于乱撞(ST/AP 不算)
  266. minimum = 2 if re.search(r'[\u4e00-\u9fff]', packed) else 4
  267. if len(packed) < minimum:
  268. return frozenset()
  269. candidates = None
  270. for gram in {packed[i:i + 2] for i in range(len(packed) - 1)}:
  271. bucket = self._rev_grams.get(gram)
  272. if not bucket:
  273. return frozenset()
  274. candidates = set(bucket) if candidates is None else candidates & bucket
  275. if not candidates:
  276. return frozenset()
  277. if not candidates:
  278. return frozenset()
  279. labels = set()
  280. for index in candidates:
  281. packed_key, key_labels = self._rev_keys[index]
  282. if packed not in packed_key:
  283. continue
  284. if key_labels <= CATCH_ALL:
  285. continue # 兜底标签不算证据
  286. # 只认"这条规则差不多就是这个品牌名"的行:留 8 个字符的余量给 牌/无中文名称 这类后缀。
  287. # 像 COMCHIPONPANJITPYSEMTECH…(多个品牌连写)这种长串一律不认。
  288. if len(packed_key) > len(packed) + 8:
  289. continue
  290. if not self._single_brand_rule(packed_key, packed):
  291. continue
  292. labels |= key_labels
  293. return frozenset(labels)
  294. def _single_brand_rule(self, packed_key, packed):
  295. """这条规则除了这段文字,还有没有写着别的已知品牌。"""
  296. rest = packed_key.replace(packed, '', 1)
  297. for piece in re.split(r'[、,,//;;||()()\s::]+', rest):
  298. piece = piece.strip()
  299. if len(piece) < 2 or re.fullmatch(r'[\d.\-]+', piece):
  300. continue
  301. found = self.alias.get(piece) or self.alias.get(piece.rstrip('牌'))
  302. if found:
  303. return False
  304. return True
  305. def brand_detail(self, raw, allow_typo=True):
  306. """Return labels, matched key, warning, method, score, ranked candidates."""
  307. if norm(raw) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(str(raw or '')):
  308. return set(), '', '', '', 0.0, ()
  309. variants = self.brand_variants(raw)
  310. whole = self.alias.get(norm(variants[0])) if variants else None
  311. if whole:
  312. return (set(whole) if len(whole) == 1 else set(), variants[0],
  313. '' if len(whole) == 1 else '品牌映射冲突', '整体精确', 100.0,
  314. tuple((v, 100.0, variants[0], '整体精确') for v in sorted(whole)))
  315. hits = []
  316. hit = ''
  317. for variant in variants:
  318. if norm(variant) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(variant):
  319. continue
  320. found = self.alias.get(norm(variant))
  321. if found:
  322. hits.append(set(found))
  323. if not hit:
  324. hit = variant
  325. if hits:
  326. agreed = set(hits[0]).intersection(*hits[1:])
  327. candidates = tuple((v, 100.0, hit, '取词精确') for v in sorted(set.union(*hits)))
  328. return (agreed if len(agreed) == 1 else set(), hit,
  329. '' if len(agreed) == 1 else '品牌映射冲突', '取词精确', 100.0, candidates)
  330. labels, hit, method, score, candidates = self.brand_matcher.match(tuple(variants), allow_typo)
  331. catch_all_hit = bool(labels) and set(labels) <= CATCH_ALL
  332. if labels and catch_all_hit and method in ('拼写近似', '包含匹配', '子串包含', '标点归一'):
  333. # 拼错一个字母就落到 Others/其他 这种兜底标签上,几乎一定是认错了
  334. # (实测 PANJIT 被拼写近似到表里的错行 PANJJIT → Others)。宁可继续往下找。
  335. labels = set()
  336. if not labels:
  337. # 反向包含:规则表里的申报内容是一长串(可能中英混写),只要这段文字
  338. # 整段出现在里面就算命中。例:'PANJIT' ⊂ 'PANJIT牌/强茂牌'。
  339. reversed_labels = None
  340. for variant in variants:
  341. text = norm(variant)
  342. # 太短的拉丁串(ST、AP)先不看;中文两字可能是完整品牌
  343. minimum = 2 if re.search(r'[\u4e00-\u9fff]', text) else 4
  344. if len(text) < minimum or re.search(r'\d', text):
  345. continue
  346. found = self.reverse_brand_labels(text)
  347. if not found:
  348. continue
  349. reversed_labels = set(found) if reversed_labels is None else reversed_labels & set(found)
  350. if reversed_labels and len(reversed_labels) == 1:
  351. return (reversed_labels, hit, '', '反向包含', 90.0,
  352. tuple((v, 90.0, hit, '反向包含') for v in sorted(reversed_labels)))
  353. if reversed_labels and len(reversed_labels) > 1:
  354. return set(), hit, '品牌反向包含冲突', '反向包含', 90.0, ()
  355. if catch_all_hit:
  356. return set(), hit, '品牌近似命中兜底标签,已忽略', method, score, candidates
  357. return (set(labels) if len(labels) == 1 else set(), hit,
  358. '品牌映射冲突' if len(labels) > 1 else '', method, score, candidates)
  359. def resolve_brand(self, raw):
  360. return self.brand_detail(raw)[:3]
  361. def is_brand(self, raw):
  362. # Avoid spelling guesses on unlabelled model IDs containing digits.
  363. labels, _, note, *_ = self.brand_detail(raw, allow_typo=not bool(re.search(r'\d', str(raw))))
  364. return bool(labels and not note)
  365. def model_inside(self, tokens, brand):
  366. """在整段文本里找回已知型号:文本包含某个已知型号即算命中,最长优先。
  367. 典型场景是“型号:K4B4G1646E等”“K4B4G1646E-BYK0”“K4B4G1646E K4A8G165WB”,
  368. 原逻辑只做整串/去“等”/空格拆分/前缀,这类型号会整条判为未识别。
  369. """
  370. best = None
  371. for token in tokens:
  372. text = norm(token)
  373. if len(text) < 4:
  374. continue
  375. candidates = set()
  376. for i in range(len(text) - 3):
  377. candidates |= self.model_grams.get(text[i:i + 4], set())
  378. for key in candidates:
  379. if key not in text:
  380. continue
  381. records = {r for r in self.models[key] if not brand or norm(r[0]) == norm(brand)}
  382. if records and (best is None or len(key) > len(best[0])):
  383. best = (key, records)
  384. return best
  385. def model_variants(self, raw):
  386. """型号候选写法:整体、括号内外、按空白/下划线、按逗号顿号枚举。"""
  387. text = self._clean(raw)
  388. text = re.sub(r'^(?:型号|MODEL)\s*[::=]\s*', '', text, flags=re.I)
  389. out = []
  390. def add(value):
  391. value = str(value or '').strip().strip('"\'“”‘’ \t').strip()
  392. if value and value not in out:
  393. out.append(value)
  394. add(text)
  395. match = re.match(r'^(.*?)[((]([^))]*)[))](.*)$', text)
  396. if match:
  397. add(match.group(1))
  398. add(match.group(2))
  399. add(match.group(1) + match.group(3))
  400. for part in re.split(r'[\s_]+', text):
  401. add(part)
  402. for part in re.split(r'[,,、]+', text):
  403. add(part)
  404. return out
  405. def app_signal(self, token):
  406. """该字段文本能否支撑一个具体应用领域(“其他/未识别”不算信号)。"""
  407. text = norm(str(token or '').replace('_', ' '))
  408. if not text or text in PLACEHOLDER:
  409. return False
  410. label = unique(self.first.get(text, ()))
  411. if label and label not in ('其他', '未识别', '通用领域'):
  412. return True
  413. return any(key in text for _, keys in self.second for key in keys)
  414. def hits_vocabulary(self, token):
  415. """该字段文本是否落在应用领域的已知词表里。"""
  416. text = norm(str(token or '').replace('_', ' '))
  417. if not text or text in PLACEHOLDER:
  418. return False
  419. if text in self.first:
  420. return True
  421. return any(key in text for _, keys in self.second for key in keys)
  422. def application(self, texts):
  423. text = norm(' '.join(texts).replace('_', ' '))
  424. if not text:
  425. return '未识别', '未识别'
  426. first = unique(self.first.get(text, ()))
  427. if not first:
  428. hits = set()
  429. for part in texts:
  430. hits.update(self.first.get(norm(part.replace('_', ' ')), ()))
  431. first = unique(hits) or ('通用领域' if hits else '其他')
  432. hits = set()
  433. for category, keys in self.second:
  434. candidate = text
  435. # Exclusions are phrase-level: satellite communication is high-tech;
  436. # computer systems are advanced computing, ordinary computers consumer.
  437. if category == '网络通信':
  438. candidate = candidate.replace('卫星通信', '')
  439. if category == '消费电子':
  440. candidate = candidate.replace('计算机系统', '')
  441. if category == '高新电子':
  442. candidate = candidate.replace('无人飞机', '')
  443. if any((bool(re.search(r'(?<![A-Z])AI(?![A-Z])', candidate)) if k == 'AI' else k in candidate) for k in keys):
  444. hits.add(category)
  445. if '通用' in text:
  446. hits.add('通用领域')
  447. return first, ('通用领域' if len(hits) > 1 else unique(hits) or '其他')
  448. @lru_cache(maxsize=8192)
  449. def parse(self, hs, raw):
  450. flags, brand_raw, model_raw, app = [], '', '', []
  451. label_brand_fallback = ''
  452. tokens, split_mode, split_warning = split_attributes(
  453. raw, is_brand=self.is_brand)
  454. if split_mode in ('space_inferred', 'underscore_inferred'):
  455. labels, _, note, kind, score, _ = self.brand_detail(raw, allow_typo=False)
  456. if labels and not note and kind == '标点归一' and score == 100:
  457. tokens, split_mode, split_warning = [str(raw)], 'primary', ''
  458. if split_warning:
  459. flags.append(split_warning)
  460. if re.search(r'(?:^|[||;;])\s*[//]\s*(?:$|[||;;])', str(raw or '')):
  461. flags.append('斜杠占位字段')
  462. schema = self.schema.get(hs, [])
  463. expected = len([x for x in schema if x])
  464. brand_slot = next((i for i, label in enumerate(schema)
  465. if label.startswith('品牌') and '类型' not in label), None)
  466. model_slot = next((i for i, label in enumerate(schema) if label == '型号'), None)
  467. first_is_status = bool(tokens and re.fullmatch('[0-4]', tokens[0]))
  468. chinese_status = bool(tokens and CHINESE_STATUS.fullmatch(tokens[0]) and
  469. brand_slot is not None and brand_slot < len(tokens) and
  470. self.is_brand(tokens[brand_slot]))
  471. extra_only_empty = len(tokens) > expected and not any(tokens[expected:])
  472. short_but_anchored = (len(tokens) < expected and brand_slot is not None and
  473. model_slot is not None and model_slot < len(tokens) and
  474. self.is_brand(tokens[brand_slot]))
  475. # Accept missing optional tail only when the brand position is independently anchored.
  476. aligned = len(tokens) == expected or extra_only_empty or short_but_anchored
  477. complete = bool(schema and aligned and
  478. (first_is_status or chinese_status))
  479. if complete:
  480. if extra_only_empty:
  481. flags.append('模板末尾空段')
  482. if short_but_anchored:
  483. flags.append('模板末尾字段缺失待核查')
  484. for label, token in zip(schema, tokens):
  485. if label.startswith('品牌') and '类型' not in label:
  486. brand_raw = token
  487. if label == '型号':
  488. model_raw = token
  489. if any(k in label for k in ['用途', '应用', '功能']):
  490. app.append(token)
  491. method = '顺序提取'
  492. # 只有"其它位置确实有真实品牌"才否定位置提取;“无/不适用”这类占位词不算线索,
  493. # 否则品牌位留空、GTIN/CAS 写“无”的记录会把已经取到的型号也一起丢掉。
  494. other_brand = any(self.is_brand(t) and norm(t) not in PLACEHOLDER for t in tokens)
  495. if brand_raw and not self.is_brand(brand_raw) and other_brand:
  496. brand_raw,model_raw,app='','',[]
  497. method='缺项识别'
  498. flags.append('顺序位置与品牌线索冲突,转缺项识别')
  499. else:
  500. method = '缺项识别'
  501. # Labels take precedence; numbered prefixes such as "1、型号:" are accepted.
  502. label_brands, label_models = [], []
  503. for index, token in enumerate(tokens):
  504. for label, value in labels_in(token):
  505. if label in ('品牌', 'BRAND'):
  506. label_brands.append(value)
  507. elif label in ('型号', 'MODEL'):
  508. label_models.append(value)
  509. else: app.append(value)
  510. # 键和值各占一段:品牌|MURATA|型号|GRM188…
  511. if KEY_TOKEN.fullmatch(token.strip()):
  512. nxt = tokens[index + 1] if index + 1 < len(tokens) else ''
  513. value = clean_value(nxt) if nxt and not KEY_TOKEN.fullmatch(nxt.strip()) else ''
  514. kind = label_kind(token.strip())
  515. if value and kind == '品牌':
  516. label_brands.append(value)
  517. elif value and kind == '型号':
  518. label_models.append(value)
  519. brand_label_conflict = len({norm(value) for value in label_brands}) > 1
  520. model_label_conflict = len({norm(value) for value in label_models}) > 1
  521. if brand_label_conflict:
  522. brand_raw = ''
  523. flags.append('品牌标签冲突')
  524. elif label_brands:
  525. brand_raw = label_brands[0]
  526. # 标签里给了品牌:能匹配上就锁死,后面的扫描不再覆盖。
  527. if NO_BRAND.fullmatch(brand_raw) or BRAND_QUALIFIER.fullmatch(brand_raw):
  528. flags.append('无品牌标记')
  529. else:
  530. probe = self.brand_detail(brand_raw)
  531. if probe[0] and not probe[2]:
  532. flags.append('品牌标签锁定')
  533. elif probe[2]:
  534. # 一句话里写了两个不同品牌:保留原文,交给后面统一判冲突,不擅自取一个。
  535. pass
  536. else:
  537. head = re.split(r'[,,;;。]', brand_raw)[0].strip()
  538. if head and head != brand_raw:
  539. head_probe = self.brand_detail(head)
  540. if head_probe[0] and not head_probe[2]:
  541. # “品牌:TDK,用于手机”这种句子值:逗号后面是说明,取前半段
  542. brand_raw = head
  543. flags.append('品牌标签锁定')
  544. probe = head_probe
  545. else:
  546. probe = ()
  547. if not probe:
  548. flags.append('品牌标签未命中')
  549. label_brand_fallback, brand_raw = brand_raw, ''
  550. if model_label_conflict:
  551. # 一条记录里同时写了“型号:”“规格型号:”等,按“多型号取首个”处理,不整条丢弃。
  552. model_raw = label_models[0]
  553. flags.append('多型号取首个')
  554. elif label_models:
  555. model_raw = label_models[0]
  556. # 明确写“无型号/无规格”的,视为已经说清楚,不再去别处凑一个型号,也不算识别失败。
  557. no_model_marker = bool(model_raw and NO_MODEL.fullmatch(model_raw))
  558. if no_model_marker:
  559. model_raw = ''
  560. flags.append('无型号标记')
  561. if not brand_raw and not brand_label_conflict:
  562. brand_raw = next((t for t in tokens if NO_BRAND.fullmatch(t)), '')
  563. if not brand_raw and not brand_label_conflict:
  564. # 逐字段按“整体 -> 去牌 -> 括号内外 -> 枚举项”解析,排除“无/不适用”等占位词。
  565. found = []
  566. for index, token in enumerate(tokens):
  567. if not token or norm(token) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(token) \
  568. or re.fullmatch(r'[\d.]+', token):
  569. continue
  570. # Explicit model/application fields cannot supply a brand by fuzzy scanning.
  571. token_labels = list(labels_in(token))
  572. if token_labels and not any(label in ('品牌', 'BRAND') for label, _ in token_labels):
  573. continue
  574. allow_typo = (index == brand_slot or
  575. (not re.search(r'\d', token) and not self.hits_vocabulary(token)))
  576. labels, hit, note, kind, score, candidates = self.brand_detail(token, allow_typo=allow_typo)
  577. if labels and not note:
  578. found.append((score, index, token, labels))
  579. if found:
  580. best = max(row[0] for row in found)
  581. top = [row for row in found if row[0] == best]
  582. distinct = set.union(*(row[3] for row in top))
  583. if len(distinct) == 1:
  584. brand_raw = top[0][2]
  585. else:
  586. flags.append('品牌位置不确定')
  587. if not brand_raw and label_brand_fallback:
  588. # 标签里写过、但匹配不上,且别处也没找到:保留原文,便于核查到底是哪种写法没覆盖。
  589. brand_raw = label_brand_fallback
  590. # “ABACUS等牌”里的“等牌”不是名字的一部分,去掉再查表,否则同品牌会被劈成两条。
  591. brand_raw, brand_had_deng = strip_brand_suffix(brand_raw)
  592. if brand_had_deng:
  593. flags.append('品牌带“等”后缀')
  594. brand_match_method, brand_match_score, brand_match_key, brand_candidates = '', 0.0, '', ()
  595. brand_set = set()
  596. if not NO_BRAND.fullmatch(brand_raw):
  597. brand_set, brand_hit, brand_note, brand_match_method, brand_match_score, brand_candidates = self.brand_detail(brand_raw)
  598. brand_match_key = brand_hit
  599. if brand_note:
  600. flags.append(brand_note)
  601. elif brand_match_method in ('标点归一', '包含匹配', '拼写近似'):
  602. flags.append('品牌模糊匹配')
  603. elif brand_match_method == '子串包含':
  604. flags.append('品牌包含匹配')
  605. elif brand_hit and brand_hit != brand_raw:
  606. flags.append('品牌括号或枚举取词')
  607. brand = unique(brand_set)
  608. if NO_BRAND.fullmatch(brand_raw) or BRAND_QUALIFIER.fullmatch(brand_raw):
  609. flags.append('无品牌标记')
  610. if not model_raw and not model_label_conflict and not no_model_marker:
  611. found = [(i, t) for i, t in enumerate(tokens) if norm(t) in self.models]
  612. if found: model_raw = found[0][1]
  613. if len(found) > 1: flags.append('多型号取首个')
  614. if not model_raw and brand_raw and not model_label_conflict and not no_model_marker:
  615. for i, token in enumerate(tokens):
  616. # 带用途/应用等标签的段不能供型号:否则“用途:5G基站”会被当成型号。
  617. token_labels = list(labels_in(token))
  618. if token_labels and not any(label in ('品牌', 'BRAND', '型号', 'MODEL')
  619. for label, _ in token_labels):
  620. continue
  621. variants = self.brand_variants(token)
  622. hit = next((v for v in variants if norm(v) == norm(brand_raw)), '')
  623. if brand_match_key and brand_match_key in token and self.resolve_brand(token)[0] == {brand}:
  624. hit = brand_match_key
  625. if not hit and brand and self.resolve_brand(token)[0] == {brand}:
  626. hit = brand_match_key if brand_match_key in token else token
  627. if not hit:
  628. continue
  629. tail = clean_value(token[token.find(hit) + len(hit):]) if hit in token else ''
  630. following = tokens[i + 1] if i + 1 < len(tokens) else ''
  631. for candidate in (tail, following):
  632. value = model_candidate(candidate, self.is_brand, self.app_signal)
  633. if value:
  634. model_raw = value
  635. flags.append('品牌后型号推断')
  636. break
  637. break
  638. if not model_raw and not model_label_conflict and not no_model_marker and brand:
  639. # 型号写在品牌前面("GRM188R71H104KA93D MURATA"):只认同品牌紧挨着的那一段,
  640. # 且必须明显像料号,避免把 100nF、0402、5G基站 这类参数或用途当成型号。
  641. anchor = next((i for i, t in enumerate(tokens)
  642. if norm(t) == norm(brand_raw)), None)
  643. if anchor is None and brand_match_key:
  644. anchor = next((i for i, t in enumerate(tokens)
  645. if brand_match_key in t), None)
  646. if anchor:
  647. cand = clean_value(tokens[anchor - 1]) if anchor > 0 else ''
  648. if (looks_like_model(cand) and cand != brand_raw
  649. and not self.is_brand(cand) and not self.app_signal(cand)
  650. and norm(cand) not in PLACEHOLDER):
  651. model_raw = cand
  652. flags.append('品牌前型号推断')
  653. if not model_raw and not model_label_conflict and not no_model_marker and brand:
  654. # 型号和品牌挤在同一段里("GRM188… MURATA"):把每段再拆成片段,挑明显像料号的那一个。
  655. for token in tokens:
  656. for part in re.split(r'[\s_\u3000、,,()()]+', token):
  657. cand = clean_value(part)
  658. if (looks_like_model(cand) and not self.is_brand(cand)
  659. and not self.app_signal(cand) and norm(cand) not in PLACEHOLDER
  660. and norm(cand) != norm(brand_raw)):
  661. model_raw = cand
  662. flags.append('型号片段提取')
  663. break
  664. if model_raw:
  665. break
  666. # “A5E30295495R2W等型”里的“等型”不是型号的一部分:去掉,并按“多型号取首个”记账。
  667. if not model_raw and not no_model_marker and not model_label_conflict:
  668. # 品牌认不出来也不能把型号一起丢掉:如果前一段明显是品牌字段(带“牌/品牌”),
  669. # 紧跟其后的那一段又像料号,就照取(型号本身不依赖品牌表)。
  670. for index, token in enumerate(tokens[:-1]):
  671. if BRAND_FIELD.search(clean_value(token) or token.strip()):
  672. nxt = tokens[index + 1]
  673. value = clean_value(nxt)
  674. if (looks_like_model(value) and value != brand_raw
  675. and not self.is_brand(value) and not self.app_signal(value)):
  676. model_raw = value
  677. flags.append('品牌字段后型号推断')
  678. break
  679. # “A5E30295495R2W等型”里的“等型”不是型号的一部分:去掉,并按“多型号取首个”记账。
  680. model_raw, model_had_deng = strip_model_suffix(model_raw)
  681. if model_had_deng:
  682. flags.append('多型号取首个')
  683. model = norm(model_raw)
  684. if model and model not in self.models:
  685. if model.endswith('等'):
  686. model = model[:-1]
  687. flags.append('多型号取首个')
  688. if any(mark in model_raw for mark in ',,、'):
  689. first = norm(re.split(r'[,,、]+', model_raw, maxsplit=1)[0])
  690. if first:
  691. model = first
  692. flags.append('多型号取首个')
  693. elif re.search(r'[\s_]', model_raw) and model not in self.models:
  694. words = [norm(part) for part in re.split(r'[\s_]+', model_raw) if part]
  695. matches_in_words = [part for part in words if part in self.models]
  696. if len(matches_in_words) == 1:
  697. model = matches_in_words[0]
  698. flags.append('空格分隔型号推断')
  699. if not model and hs in MEMORY_HS and not model_label_conflict and not no_model_marker:
  700. # 型号可能和品牌挤在同一段(如“长江存储(YMTC) YMTC-XX”),逐字段找回。
  701. for token in tokens:
  702. for variant in self.model_variants(token):
  703. key = norm(variant)
  704. if not key or key in PLACEHOLDER or key not in self.models:
  705. continue
  706. hits = self.models[key]
  707. if brand:
  708. hits = {r for r in hits if norm(r[0]) == norm(brand)}
  709. if len(hits) == 1:
  710. model_raw, model = variant, key
  711. flags.append('型号候选扫描')
  712. break
  713. if model:
  714. break
  715. matches = self.models.get(model, set())
  716. if brand: matches = {r for r in matches if norm(r[0]) == norm(brand)}
  717. if not matches and model and hs in MEMORY_HS:
  718. for length in range(len(model),0,-1):
  719. hits={r for r in self.prefix_lookup.get(model[:length],()) if not brand or norm(r[0])==norm(brand)}
  720. if hits:
  721. matches=hits;flags.append('型号前缀匹配');break
  722. # 最后兜底:文本里含某个已知型号就算命中(“xxx等”“xxx-yyy”)。
  723. # 只在型号表覆盖的 HS 上做,避免把无关商品的编码当型号。
  724. if not matches and hs in MEMORY_HS and self.fuzzy_contains:
  725. hit = self.model_inside(tokens, brand)
  726. if hit:
  727. model, matches = hit
  728. flags.append('型号包含匹配')
  729. # Different descriptions that imply the same taxonomy are still a mapping conflict.
  730. mapped = next(iter(matches)) if len(matches) == 1 else None
  731. if len(matches) > 1: flags.append('型号映射冲突')
  732. if mapped and not brand and not any('品牌' in f and ('冲突' in f or '不确定' in f) for f in flags):
  733. brand = mapped[0]
  734. brand_match_method, brand_match_score = '型号反推', 0.0
  735. if not brand and not NO_BRAND.fullmatch(brand_raw) and not BRAND_QUALIFIER.fullmatch(brand_raw):
  736. flags.append('品牌未识别')
  737. # 字段里有像品牌名的拉丁写法、但映射表查不到(如 ABACUS / PANJIT):
  738. # 明确标出来,用户照着补一行规则就能识别,不用逐条猜。
  739. for token in tokens:
  740. if looks_like_model(token) or re.search(r'\d', token):
  741. continue # 含数字的长串多半是型号,别拿它的碎片当品牌猜
  742. for part in re.split(r'[//、,,;;||\s]+', token):
  743. candidate, _ = strip_brand_suffix(clean_value(part))
  744. if (LATIN_ONLY.fullmatch(candidate) and len(candidate) >= 3
  745. and not re.search(r'\d', candidate)
  746. and norm(candidate) not in PLACEHOLDER
  747. and not self.is_brand(candidate)):
  748. flags.append(f'疑似缺品牌:{candidate}')
  749. break
  750. else:
  751. continue
  752. break
  753. if not model and not no_model_marker: flags.append('型号未识别')
  754. def has_signal(values):
  755. return any(self.app_signal(value) for value in values)
  756. if not has_signal(app):
  757. # 缺项识别时位置仍可参考:前两位是品牌类型/出口享惠,应用场景与功能紧随其后。
  758. excluded = {brand_raw, model_raw}
  759. for slot in (2, 3):
  760. if slot < len(tokens):
  761. token = tokens[slot]
  762. if token and token not in excluded and norm(token) not in PLACEHOLDER \
  763. and not re.fullmatch(r'[\d.]+', token) and self.app_signal(token) \
  764. and token not in app:
  765. app.append(token)
  766. if not has_signal(app):
  767. app.extend(t for t in tokens if any(k in t for k in ['用', '应用', '功能'])
  768. and t not in (brand_raw, model_raw) and t not in app)
  769. if not has_signal(app):
  770. app.extend(t for t in tokens if t and t not in (brand_raw, model_raw)
  771. and t not in app and self.hits_vocabulary(t))
  772. a1, a2 = self.application(app)
  773. if a1 != a2: flags.append('应用二轮变更')
  774. if a2 in ('其他', '未识别'): flags.append('应用未识别')
  775. # 只有落在项目范围内的 HS 才有类目;范围外的记录留空,避免把无关 HS 误标成存储器。
  776. c1, c2, c3, c4, description = '', '', '', '', ''
  777. if hs in self.cap:
  778. c1, c2 = '电容器', self.cap[hs]
  779. elif hs in MEMORY_HS:
  780. c1, c2 = '存储器', '其他存储器'
  781. if mapped and mapped[5]=='存储器':
  782. description = mapped[1]
  783. content = norm(' '.join(mapped[1:]))
  784. generation = re.search(r'(?:LP)?DDR([345])', content)
  785. if 'DDR' in content:
  786. c2, c3, c4 = 'DRAM', 'DDR', 'DDR' + generation[1] if generation else '其他DDR'
  787. elif 'SDRAM' in content: c2, c3 = 'DRAM', 'SDRAM'
  788. elif 'DRAM' in content or 'HBM' in content: c2, c3 = 'DRAM', '其他DRAM'
  789. elif 'FLASH' in content or '闪存' in content:
  790. c2, c3 = 'Flash', 'NAND' if 'NAND' in content else 'NOR' if 'NOR' in content else '其他Flash'
  791. if hs in MEMORY_HS and not mapped: flags.append('存储器分类未识别')
  792. country = unique(self.brand_country.get(norm(brand), ())) or '未识别'
  793. return dict(brand=brand, model=model, brand_country=country, c1=c1, c2=c2, c3=c3, c4=c4,
  794. description=description, app1=a1, app=a2, method=method, flags=';'.join(sorted(set(flags))),
  795. app_raw=' '.join(app), brand_raw=brand_raw, model_raw=model_raw, mapped=int(mapped is not None),
  796. brand_match_method=brand_match_method, brand_match_score=round(brand_match_score, 6),
  797. brand_match_key=brand_match_key, brand_candidates=brand_candidates)
  798. def application_order(labels) -> list[str]:
  799. """把任意标签集合按固定顺序排序,未列出的排最后。"""
  800. order = {label: index for index, label in enumerate(APPLICATION_LABELS)}
  801. return sorted(labels, key=lambda label: (order.get(label, 99), str(label)))
  802. APPLICATION_LABELS = ['先进计算', '汽车电子', '工业应用', '网络通信', '新能源', '高新电子', '消费电子', '通用领域', '其他']